Return-Path: X-Original-To: apmail-cassandra-commits-archive@www.apache.org Delivered-To: apmail-cassandra-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B3072188E3 for ; Tue, 16 Jun 2015 15:40:59 +0000 (UTC) Received: (qmail 17122 invoked by uid 500); 16 Jun 2015 15:40:54 -0000 Delivered-To: apmail-cassandra-commits-archive@cassandra.apache.org Received: (qmail 17086 invoked by uid 500); 16 Jun 2015 15:40:54 -0000 Mailing-List: contact commits-help@cassandra.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cassandra.apache.org Delivered-To: mailing list commits@cassandra.apache.org Received: (qmail 17071 invoked by uid 99); 16 Jun 2015 15:40:54 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 16 Jun 2015 15:40:54 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 695E5E027F; Tue, 16 Jun 2015 15:40:54 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: benedict@apache.org To: commits@cassandra.apache.org Date: Tue, 16 Jun 2015 15:40:54 -0000 Message-Id: <84cde8f621f94c2f8d1af9fcdec7a957@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [01/10] cassandra git commit: Periodically submit background compaction tasks Repository: cassandra Updated Branches: refs/heads/cassandra-2.0 80d46b8ac -> ec52e77ec refs/heads/cassandra-2.1 83944f82b -> 374ef3c7a refs/heads/cassandra-2.2 1fbdba2c4 -> 2c9b82d11 refs/heads/trunk be6cb9d97 -> 39680ee87 Periodically submit background compaction tasks patch by benedict; reviewed by yukim for CASSANDRA-9592 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ec52e77e Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ec52e77e Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ec52e77e Branch: refs/heads/cassandra-2.0 Commit: ec52e77ecde749e7c5a483b26cbd8041f2a5a33c Parents: 80d46b8 Author: Benedict Elliott Smith Authored: Tue Jun 16 16:28:52 2015 +0100 Committer: Benedict Elliott Smith Committed: Tue Jun 16 16:28:52 2015 +0100 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../apache/cassandra/db/ColumnFamilyStore.java | 24 ++++++++++++++++++++ .../db/compaction/CompactionManager.java | 7 +++++- .../cassandra/service/CassandraDaemon.java | 19 +++------------- 4 files changed, 34 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/ec52e77e/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 7f507c2..6d031f6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 2.0.16: + * Periodically submit background compaction tasks (CASSANDRA-9592) * Set HAS_MORE_PAGES flag to false when PagingState is null (CASSANDRA-9571) * Backport indexed value validation fix from CASSANDRA-9057 (CASSANDRA-9564) * Don't accumulate more range than necessary in RangeTombstone.Tracker (CASSANDRA-9486) http://git-wip-us.apache.org/repos/asf/cassandra/blob/ec52e77e/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index f81ec82..4fc2f24 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -174,6 +174,30 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean } } + public static Runnable getBackgroundCompactionTaskSubmitter() + { + return new Runnable() + { + public void run() + { + List submitted = new ArrayList<>(); + for (Keyspace keyspace : Keyspace.all()) + for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores()) + if (!CompactionManager.instance.submitBackground(cfs, false).isEmpty()) + submitted.add(cfs); + + while (!submitted.isEmpty() && CompactionManager.instance.getActiveCompactions() < CompactionManager.instance.getMaximumCompactorThreads()) + { + List submitMore = ImmutableList.copyOf(submitted); + submitted.clear(); + for (ColumnFamilyStore cfs : submitMore) + if (!CompactionManager.instance.submitBackground(cfs, false).isEmpty()) + submitted.add(cfs); + } + } + }; + } + public void setCompactionStrategyClass(String compactionStrategyClass) { try http://git-wip-us.apache.org/repos/asf/cassandra/blob/ec52e77e/src/java/org/apache/cassandra/db/compaction/CompactionManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java index 207b90d..c66eeb6 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java @@ -128,6 +128,11 @@ public class CompactionManager implements CompactionManagerMBean */ public List> submitBackground(final ColumnFamilyStore cfs) { + return submitBackground(cfs, true); + } + + public List> submitBackground(final ColumnFamilyStore cfs, boolean autoFill) + { if (cfs.isAutoCompactionDisabled()) { logger.debug("Autocompaction is disabled"); @@ -153,7 +158,7 @@ public class CompactionManager implements CompactionManagerMBean compactingCF.add(cfs); futures.add(executor.submit(new BackgroundCompactionTask(cfs))); // if we have room for more compactions, then fill up executor - } while (executor.getActiveCount() + futures.size() < executor.getMaximumPoolSize()); + } while (autoFill && executor.getActiveCount() + futures.size() < executor.getMaximumPoolSize()); return futures; } http://git-wip-us.apache.org/repos/asf/cassandra/blob/ec52e77e/src/java/org/apache/cassandra/service/CassandraDaemon.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java index 737f4bf..5bffb49 100644 --- a/src/java/org/apache/cassandra/service/CassandraDaemon.java +++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java @@ -411,22 +411,9 @@ public class CassandraDaemon } } } - // start compactions in five minutes (if no flushes have occurred by then to do so) - Runnable runnable = new Runnable() - { - public void run() - { - for (Keyspace keyspaceName : Keyspace.all()) - { - for (ColumnFamilyStore cf : keyspaceName.getColumnFamilyStores()) - { - for (ColumnFamilyStore store : cf.concatWithIndexes()) - CompactionManager.instance.submitBackground(store); - } - } - } - }; - StorageService.optionalTasks.schedule(runnable, 5 * 60, TimeUnit.SECONDS); + // schedule periodic background compaction task submission. this is simply a backstop against compactions stalling + // due to scheduling errors or race conditions + StorageService.optionalTasks.scheduleWithFixedDelay(ColumnFamilyStore.getBackgroundCompactionTaskSubmitter(), 5, 1, TimeUnit.MINUTES); SystemKeyspace.finishStartup();