Return-Path: X-Original-To: apmail-cassandra-commits-archive@www.apache.org Delivered-To: apmail-cassandra-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 810C518E60 for ; Mon, 17 Aug 2015 06:57:10 +0000 (UTC) Received: (qmail 98226 invoked by uid 500); 17 Aug 2015 06:56:36 -0000 Delivered-To: apmail-cassandra-commits-archive@cassandra.apache.org Received: (qmail 98185 invoked by uid 500); 17 Aug 2015 06:56:36 -0000 Mailing-List: contact commits-help@cassandra.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cassandra.apache.org Delivered-To: mailing list commits@cassandra.apache.org Received: (qmail 98142 invoked by uid 99); 17 Aug 2015 06:56:36 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 17 Aug 2015 06:56:36 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3E542E00CC; Mon, 17 Aug 2015 06:56:36 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: marcuse@apache.org To: commits@cassandra.apache.org Date: Mon, 17 Aug 2015 06:56:36 -0000 Message-Id: <01db48fbd4a448ac8e4ecd35837da5c2@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/5] cassandra git commit: Make getFullyExpiredSSTables less expensive. Repository: cassandra Updated Branches: refs/heads/trunk c41412a7f -> 6ae27a320 Make getFullyExpiredSSTables less expensive. Patch by marcuse; reviewed by yukim for CASSANDRA-9882 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f53aacba Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f53aacba Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f53aacba Branch: refs/heads/trunk Commit: f53aacba80de3c3441e1524be9498f6d15c5c411 Parents: 50e5802 Author: Marcus Eriksson Authored: Mon Aug 17 08:29:45 2015 +0200 Committer: Marcus Eriksson Committed: Mon Aug 17 08:29:45 2015 +0200 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../apache/cassandra/db/ColumnFamilyStore.java | 55 +++++++++++++++++--- .../DateTieredCompactionStrategy.java | 13 ++++- .../DateTieredCompactionStrategyTest.java | 1 + 4 files changed, 60 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/f53aacba/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 7d84538..905445e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 2.0.17 + * Make getFullyExpiredSSTables less expensive (CASSANDRA-9882) * Add tool to find why expired sstables are not getting dropped (CASSANDRA-10015) * Remove erroneous pending HH tasks from tpstats/jmx (CASSANDRA-9129) * Don't cast expected bf size to an int (CASSANDRA-9959) http://git-wip-us.apache.org/repos/asf/cassandra/blob/f53aacba/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index c125cf0..eb688f7 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -1028,17 +1028,56 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean if (sstables.isEmpty()) return ImmutableSet.of(); - DataTracker.SSTableIntervalTree tree = data.getView().intervalTree; - - Set results = null; - for (SSTableReader sstable : sstables) + List sortedByFirst = new ArrayList<>(sstables); + Collections.sort(sortedByFirst, new Comparator() { - Set overlaps = ImmutableSet.copyOf(tree.search(Interval.create(sstable.first, sstable.last))); - results = results == null ? overlaps : Sets.union(results, overlaps).immutableCopy(); + @Override + public int compare(SSTableReader o1, SSTableReader o2) + { + return o1.first.compareTo(o2.first); + } + }); + List> intervals = new ArrayList<>(); + DecoratedKey first = null, last = null; + /* + normalize the intervals covered by the sstables + assume we have sstables like this (brackets representing first/last key in the sstable); + [ ] [ ] [ ] [ ] + [ ] [ ] + then we can, instead of searching the interval tree 6 times, normalize the intervals and + only query the tree 2 times, for these intervals; + [ ] [ ] + */ + for (SSTableReader sstable : sortedByFirst) + { + if (first == null) + { + first = sstable.first; + last = sstable.last; + } + else + { + if (sstable.first.compareTo(last) <= 0) // we do overlap + { + if (sstable.last.compareTo(last) > 0) + last = sstable.last; + } + else + { + intervals.add(Interval.create(first, last)); + first = sstable.first; + last = sstable.last; + } + } } - results = Sets.difference(results, ImmutableSet.copyOf(sstables)); + intervals.add(Interval.create(first, last)); + DataTracker.SSTableIntervalTree tree = data.getView().intervalTree; + Set results = new HashSet<>(); + + for (Interval interval : intervals) + results.addAll(tree.search(interval)); - return results; + return Sets.difference(results, ImmutableSet.copyOf(sstables)); } /** http://git-wip-us.apache.org/repos/asf/cassandra/blob/f53aacba/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java index df28bd4..fea4995 100644 --- a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java @@ -18,6 +18,7 @@ package org.apache.cassandra.db.compaction; import java.util.*; +import java.util.concurrent.TimeUnit; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Predicate; @@ -37,6 +38,8 @@ public class DateTieredCompactionStrategy extends AbstractCompactionStrategy protected DateTieredCompactionStrategyOptions options; protected volatile int estimatedRemainingTasks; + @VisibleForTesting + long lastExpiredCheck; public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map options) { @@ -83,8 +86,14 @@ public class DateTieredCompactionStrategy extends AbstractCompactionStrategy Set uncompacting = cfs.getUncompactingSSTables(); - // Find fully expired SSTables. Those will be included no matter what. - Set expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore); + Set expired = Collections.emptySet(); + // we only check for expired sstables every 10 minutes due to it being an expensive operation + if (System.currentTimeMillis() - lastExpiredCheck > TimeUnit.MINUTES.toMillis(10)) + { + // Find fully expired SSTables. Those will be included no matter what. + expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore); + lastExpiredCheck = System.currentTimeMillis(); + } Set candidates = Sets.newHashSet(filterSuspectSSTables(uncompacting)); List compactionCandidates = new ArrayList<>(getNextNonExpiredSSTables(Sets.difference(candidates, expired), gcBefore)); http://git-wip-us.apache.org/repos/asf/cassandra/blob/f53aacba/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java index d7caf6b..0084a16 100644 --- a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java @@ -308,6 +308,7 @@ public class DateTieredCompactionStrategyTest extends SchemaLoader DateTieredCompactionStrategy dtcs = new DateTieredCompactionStrategy(cfs, options); dtcs.startup(); assertNull(dtcs.getNextBackgroundTask((int) (System.currentTimeMillis() / 1000))); + dtcs.lastExpiredCheck = 0; Thread.sleep(2000); AbstractCompactionTask t = dtcs.getNextBackgroundTask((int) (System.currentTimeMillis()/1000)); assertNotNull(t);