cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marc...@apache.org
Subject [3/3] cassandra git commit: Merge branch 'cassandra-2.1' into cassandra-2.2
Date Tue, 11 Aug 2015 08:26:09 GMT
Merge branch 'cassandra-2.1' into cassandra-2.2

Conflicts:
	test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/5431a88d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/5431a88d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/5431a88d

Branch: refs/heads/cassandra-2.2
Commit: 5431a88d7bd77ee5c2b7938dd9b014bf623979cd
Parents: 4f14c85 012b987
Author: Marcus Eriksson <marcuse@apache.org>
Authored: Tue Aug 11 10:18:03 2015 +0200
Committer: Marcus Eriksson <marcuse@apache.org>
Committed: Tue Aug 11 10:18:03 2015 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../cassandra/tools/SSTableExpiredBlockers.java | 136 +++++++++++++++++++
 .../cassandra/db/compaction/TTLExpiryTest.java  |  30 +++-
 tools/bin/sstableexpiredblockers                |  54 ++++++++
 tools/bin/sstableexpiredblockers.bat            |  23 ++++
 5 files changed, 243 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/5431a88d/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 772455c,f7fb63c..cff477b
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -37,9 -16,15 +37,10 @@@ Merged from 2.1
   * Fix clientutil jar and tests (CASSANDRA-9760)
   * (cqlsh) Allow the SSL protocol version to be specified through the
     config file or environment variables (CASSANDRA-9544)
 - * Remove repair snapshot leftover on startup (CASSANDRA-7357)
 - * Use random nodes for batch log when only 2 racks (CASSANDRA-8735)
 - * Ensure atomicity inside thrift and stream session (CASSANDRA-7757)
 - * Fix nodetool info error when the node is not joined (CASSANDRA-9031)
  Merged from 2.0:
+  * Add tool to find why expired sstables are not getting dropped (CASSANDRA-10015)
   * Remove erroneous pending HH tasks from tpstats/jmx (CASSANDRA-9129)
   * Don't cast expected bf size to an int (CASSANDRA-9959)
 - * Log when messages are dropped due to cross_node_timeout (CASSANDRA-9793)
   * checkForEndpointCollision fails for legitimate collisions (CASSANDRA-9765)
   * Complete CASSANDRA-8448 fix (CASSANDRA-9519)
   * Don't include auth credentials in debug log (CASSANDRA-9682)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5431a88d/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
index 0000000,b4f7063..2feee76
mode 000000,100644..100644
--- a/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
+++ b/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
@@@ -1,0 -1,136 +1,136 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ package org.apache.cassandra.tools;
+ 
+ import java.io.IOException;
+ import java.io.PrintStream;
+ import java.util.Collections;
+ import java.util.HashSet;
+ import java.util.Map;
+ import java.util.Set;
+ 
+ import com.google.common.base.Throwables;
+ import com.google.common.collect.ArrayListMultimap;
+ import com.google.common.collect.Multimap;
+ 
+ import org.apache.cassandra.config.CFMetaData;
+ import org.apache.cassandra.config.DatabaseDescriptor;
+ import org.apache.cassandra.config.Schema;
+ import org.apache.cassandra.db.ColumnFamilyStore;
+ import org.apache.cassandra.db.Directories;
+ import org.apache.cassandra.db.Keyspace;
+ import org.apache.cassandra.io.sstable.Component;
+ import org.apache.cassandra.io.sstable.Descriptor;
 -import org.apache.cassandra.io.sstable.SSTableReader;
++import org.apache.cassandra.io.sstable.format.SSTableReader;
+ 
+ /**
+  * During compaction we can drop entire sstables if they only contain expired tombstones
and if it is guaranteed
+  * to not cover anything in other sstables. An expired sstable can be blocked from getting
dropped if its newest
+  * timestamp is newer than the oldest data in another sstable.
+  *
+  * This class outputs all sstables that are blocking other sstables from getting dropped
so that a user can
+  * figure out why certain sstables are still on disk.
+  */
+ public class SSTableExpiredBlockers
+ {
+     public static void main(String[] args) throws IOException
+     {
+         PrintStream out = System.out;
+         if (args.length < 2)
+         {
+             out.println("Usage: sstableexpiredblockers <keyspace> <table>");
+             System.exit(1);
+         }
+         String keyspace = args[args.length - 2];
+         String columnfamily = args[args.length - 1];
 -        DatabaseDescriptor.loadSchemas();
++        Schema.instance.loadFromDisk(false);
+ 
+         CFMetaData metadata = Schema.instance.getCFMetaData(keyspace, columnfamily);
+         if (metadata == null)
+             throw new IllegalArgumentException(String.format("Unknown keyspace/table %s.%s",
+                     keyspace,
+                     columnfamily));
+ 
+         Keyspace ks = Keyspace.openWithoutSSTables(keyspace);
+         ColumnFamilyStore cfs = ks.getColumnFamilyStore(columnfamily);
+         Directories.SSTableLister lister = cfs.directories.sstableLister().skipTemporary(true);
+         Set<SSTableReader> sstables = new HashSet<>();
+         for (Map.Entry<Descriptor, Set<Component>> sstable : lister.list().entrySet())
+         {
+             if (sstable.getKey() != null)
+             {
+                 try
+                 {
+                     SSTableReader reader = SSTableReader.open(sstable.getKey());
+                     sstables.add(reader);
+                 }
+                 catch (Throwable t)
+                 {
+                     out.println("Couldn't open sstable: " + sstable.getKey().filenameFor(Component.DATA));
+                     Throwables.propagate(t);
+                 }
+             }
+         }
+         if (sstables.isEmpty())
+         {
+             out.println("No sstables for " + keyspace + "." + columnfamily);
+             System.exit(1);
+         }
+ 
+         int gcBefore = (int)(System.currentTimeMillis()/1000) - metadata.getGcGraceSeconds();
+         Multimap<SSTableReader, SSTableReader> blockers = checkForExpiredSSTableBlockers(sstables,
gcBefore);
+         for (SSTableReader blocker : blockers.keySet())
+         {
+             out.println(String.format("%s blocks %d expired sstables from getting dropped:
%s%n",
+                                     formatForExpiryTracing(Collections.singleton(blocker)),
+                                     blockers.get(blocker).size(),
+                                     formatForExpiryTracing(blockers.get(blocker))));
+         }
+ 
+         System.exit(0);
+     }
+ 
+     public static Multimap<SSTableReader, SSTableReader> checkForExpiredSSTableBlockers(Iterable<SSTableReader>
sstables, int gcBefore)
+     {
+         Multimap<SSTableReader, SSTableReader> blockers = ArrayListMultimap.create();
+         for (SSTableReader sstable : sstables)
+         {
+             if (sstable.getSSTableMetadata().maxLocalDeletionTime < gcBefore)
+             {
+                 for (SSTableReader potentialBlocker : sstables)
+                 {
+                     if (!potentialBlocker.equals(sstable) &&
+                         potentialBlocker.getMinTimestamp() <= sstable.getMaxTimestamp()
&&
+                         potentialBlocker.getSSTableMetadata().maxLocalDeletionTime >
gcBefore)
+                         blockers.put(potentialBlocker, sstable);
+                 }
+             }
+         }
+         return blockers;
+     }
+ 
+     private static String formatForExpiryTracing(Iterable<SSTableReader> sstables)
+     {
+         StringBuilder sb = new StringBuilder();
+ 
+         for (SSTableReader sstable : sstables)
+             sb.append(String.format("[%s (minTS = %d, maxTS = %d, maxLDT = %d)]", sstable,
sstable.getMinTimestamp(), sstable.getMaxTimestamp(), sstable.getSSTableMetadata().maxLocalDeletionTime)).append(",
");
+ 
+         return sb.toString();
+     }
+ }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5431a88d/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
index 579794d,a5b376e..bd1e559
--- a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
@@@ -20,8 -20,7 +20,8 @@@ package org.apache.cassandra.db.compact
   * 
   */
  
- import org.apache.cassandra.io.sstable.format.SSTableReader;
 +import org.junit.BeforeClass;
+ import com.google.common.collect.Multimap;
  import com.google.common.collect.Sets;
  import org.junit.Test;
  import org.junit.runner.RunWith;
@@@ -29,15 -28,13 +29,17 @@@
  import org.apache.cassandra.OrderedJUnit4ClassRunner;
  import org.apache.cassandra.SchemaLoader;
  import org.apache.cassandra.Util;
 +import org.apache.cassandra.config.KSMetaData;
  import org.apache.cassandra.db.*;
  import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
 +import org.apache.cassandra.exceptions.ConfigurationException;
++import org.apache.cassandra.io.sstable.format.SSTableReader;
  import org.apache.cassandra.io.sstable.ISSTableScanner;
 -import org.apache.cassandra.io.sstable.SSTableReader;
 +import org.apache.cassandra.locator.SimpleStrategy;
+ import org.apache.cassandra.tools.SSTableExpiredBlockers;
  import org.apache.cassandra.utils.ByteBufferUtil;
  
 +import java.io.IOException;
  import java.util.Collections;
  import java.util.Set;
  
@@@ -206,7 -190,32 +208,33 @@@ public class TTLExpiryTes
              OnDiskAtomIterator iter = scanner.next();
              assertEquals(noTTLKey, iter.getKey());
          }
 +
 +        scanner.close();
      }
+ 
+     @Test
+     public void testCheckForExpiredSSTableBlockers() throws InterruptedException
+     {
 -        String KEYSPACE1 = "Keyspace1";
 -        ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
++        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1");
+         cfs.truncateBlocking();
+         cfs.disableAutoCompaction();
+         cfs.metadata.gcGraceSeconds(0);
+ 
+         Mutation rm = new Mutation(KEYSPACE1, Util.dk("test").getKey());
+         rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis());
+         rm.applyUnsafe();
+         cfs.forceBlockingFlush();
+         SSTableReader blockingSSTable = cfs.getSSTables().iterator().next();
+         for (int i = 0; i < 10; i++)
+         {
+             rm = new Mutation(KEYSPACE1, Util.dk("test").getKey());
+             rm.delete("Standard1", System.currentTimeMillis());
+             rm.applyUnsafe();
+             cfs.forceBlockingFlush();
+         }
+         Multimap<SSTableReader, SSTableReader> blockers = SSTableExpiredBlockers.checkForExpiredSSTableBlockers(cfs.getSSTables(),
(int) (System.currentTimeMillis() / 1000) + 100);
+         assertEquals(1, blockers.keySet().size());
+         assertTrue(blockers.keySet().contains(blockingSSTable));
+         assertEquals(10, blockers.get(blockingSSTable).size());
+     }
  }


Mime
View raw message