Return-Path: X-Original-To: apmail-cassandra-commits-archive@www.apache.org Delivered-To: apmail-cassandra-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 62E3D10FFD for ; Thu, 29 Aug 2013 17:53:09 +0000 (UTC) Received: (qmail 35823 invoked by uid 500); 29 Aug 2013 17:53:08 -0000 Delivered-To: apmail-cassandra-commits-archive@cassandra.apache.org Received: (qmail 35779 invoked by uid 500); 29 Aug 2013 17:53:08 -0000 Mailing-List: contact commits-help@cassandra.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cassandra.apache.org Delivered-To: mailing list commits@cassandra.apache.org Received: (qmail 35757 invoked by uid 99); 29 Aug 2013 17:53:08 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 29 Aug 2013 17:53:08 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id B310F8C6CE0; Thu, 29 Aug 2013 17:53:07 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jbellis@apache.org To: commits@cassandra.apache.org Date: Thu, 29 Aug 2013 17:53:08 -0000 Message-Id: <5395902b7b994831ac40b60d0b60b0df@git.apache.org> In-Reply-To: <1802191ed5334bf4a78e9b6a7a93585f@git.apache.org> References: <1802191ed5334bf4a78e9b6a7a93585f@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/3] git commit: Improve leveled compaction's ability to find non-overlapping L0 compactions to work on concurrently patch by marcuse and jbellis; reviewed by yukim for CASSANDRA-5921 Improve leveled compaction's ability to find non-overlapping L0 compactions to work on concurrently patch by marcuse and jbellis; reviewed by yukim for CASSANDRA-5921 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/c1e0f310 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/c1e0f310 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/c1e0f310 Branch: refs/heads/trunk Commit: c1e0f3102ebd5670927ce89420681971b8325379 Parents: 9495eb5 Author: Jonathan Ellis Authored: Thu Aug 29 12:51:58 2013 -0500 Committer: Jonathan Ellis Committed: Thu Aug 29 12:51:58 2013 -0500 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ .../db/compaction/LeveledManifest.java | 20 +++++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/c1e0f310/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 8a1004f..2d3ee24 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,6 @@ 2.0.1 + * Improve leveled compaction's ability to find non-overlapping L0 compactions + to work on concurrently (CASSANDRA-5921) * Notify indexer of columns shadowed by range tombstones (CASSANDRA-5614) * Log Merkle tree stats (CASSANDRA-2698) * Switch from crc32 to adler32 for compressed sstable checksums (CASSANDRA-5862) http://git-wip-us.apache.org/repos/asf/cassandra/blob/c1e0f310/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java index 597b851..bc6824a 100644 --- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java +++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java @@ -25,6 +25,7 @@ import java.util.*; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Predicate; import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; @@ -412,6 +413,8 @@ public class LeveledManifest if (level == 0) { + Set compactingL0 = ImmutableSet.copyOf(Iterables.filter(generations[0], Predicates.in(compacting))); + // L0 is the dumping ground for new sstables which thus may overlap each other. // // We treat L0 compactions specially: @@ -433,13 +436,14 @@ public class LeveledManifest if (candidates.contains(sstable)) continue; - for (SSTableReader newCandidate : Sets.union(Collections.singleton(sstable), overlapping(sstable, remaining))) + Sets.SetView overlappedL0 = Sets.union(Collections.singleton(sstable), overlapping(sstable, remaining)); + if (!Sets.intersection(overlappedL0, compactingL0).isEmpty()) + continue; + + for (SSTableReader newCandidate : overlappedL0) { - if (!compacting.contains(newCandidate)) - { - candidates.add(newCandidate); - remaining.remove(newCandidate); - } + candidates.add(newCandidate); + remaining.remove(newCandidate); } if (candidates.size() > MAX_COMPACTING_L0) @@ -458,9 +462,7 @@ public class LeveledManifest // TODO try to find a set of L0 sstables that only overlaps with non-busy L1 sstables candidates = Sets.union(candidates, overlapping(candidates, generations[1])); } - // check overlap with L0 compacting sstables to make sure we are not generating overlap in L1. - Iterable compactingL0 = Iterables.filter(generations[0], Predicates.in(compacting)); - if (candidates.size() < 2 || !Sets.intersection(candidates, compacting).isEmpty() || !overlapping(candidates, compactingL0).isEmpty()) + if (candidates.size() < 2) return Collections.emptyList(); else return candidates;