Return-Path: X-Original-To: apmail-cassandra-commits-archive@www.apache.org Delivered-To: apmail-cassandra-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id ECDBE9D65 for ; Tue, 7 Feb 2012 15:09:34 +0000 (UTC) Received: (qmail 28858 invoked by uid 500); 7 Feb 2012 15:09:34 -0000 Delivered-To: apmail-cassandra-commits-archive@cassandra.apache.org Received: (qmail 28839 invoked by uid 500); 7 Feb 2012 15:09:34 -0000 Mailing-List: contact commits-help@cassandra.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cassandra.apache.org Delivered-To: mailing list commits@cassandra.apache.org Received: (qmail 28831 invoked by uid 99); 7 Feb 2012 15:09:34 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 07 Feb 2012 15:09:34 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.114] (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 07 Feb 2012 15:09:32 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 7920631D176; Tue, 7 Feb 2012 15:09:12 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: xedin@apache.org To: commits@cassandra.apache.org X-Mailer: ASF-Git Admin Mailer Subject: git commit: fix truncate not to leave behind non-CFS backed secondary indexes patch by Pavel Yaskevich; reviewed by Jake Luciani for CASSANDRA-3844 Message-Id: <20120207150912.7920631D176@tyr.zones.apache.org> Date: Tue, 7 Feb 2012 15:09:12 +0000 (UTC) Updated Branches: refs/heads/cassandra-1.0 3df7040fc -> 545288357 fix truncate not to leave behind non-CFS backed secondary indexes patch by Pavel Yaskevich; reviewed by Jake Luciani for CASSANDRA-3844 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/54528835 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/54528835 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/54528835 Branch: refs/heads/cassandra-1.0 Commit: 54528835775460f427989138529624f8a6b6e34c Parents: 3df7040 Author: Pavel Yaskevich Authored: Tue Feb 7 17:00:23 2012 +0300 Committer: Pavel Yaskevich Committed: Tue Feb 7 18:00:45 2012 +0300 ---------------------------------------------------------------------- CHANGES.txt | 2 + .../org/apache/cassandra/db/ColumnFamilyStore.java | 20 +++++++++++++++ .../cassandra/db/compaction/CompactionManager.java | 17 ++++-------- .../apache/cassandra/db/index/SecondaryIndex.java | 8 +++++- .../cassandra/db/index/SecondaryIndexManager.java | 16 +++++++++++- .../apache/cassandra/db/index/keys/KeysIndex.java | 5 +++ 6 files changed, 55 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/54528835/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 83eca8a..27831ba 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -11,6 +11,8 @@ * synchronize BiMap of bootstrapping tokens (CASSANDRA-3417) * show index options in CLI (CASSANDRA-3809) * add optional socket timeout for streaming (CASSANDRA-3838) + * fix truncate not to leave behind non-CFS backed secondary indexes + (CASSANDRA-3844) Merged from 0.8: * (Pig) fix CassandraStorage to use correct comparator in Super ColumnFamily case (CASSANDRA-3251) http://git-wip-us.apache.org/repos/asf/cassandra/blob/54528835/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 151564c..0952ffd 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -1939,4 +1939,24 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean DataTracker.View view = data.getView(); return view.sstables.isEmpty() && view.memtable.getOperations() == 0 && view.memtablesPendingFlush.isEmpty(); } + + /** + * Discard all SSTables that were created before given timestamp. Caller is responsible to obtain compactionLock. + * + * @param truncatedAt The timestamp of the truncation + * (all SSTables before that timestamp are going be marked as compacted) + */ + public void discardSSTables(long truncatedAt) + { + List truncatedSSTables = new ArrayList(); + + for (SSTableReader sstable : getSSTables()) + { + if (!sstable.newSince(truncatedAt)) + truncatedSSTables.add(sstable); + } + + if (!truncatedSSTables.isEmpty()) + markCompacted(truncatedSSTables); + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/54528835/src/java/org/apache/cassandra/db/compaction/CompactionManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java index ac25c4b..caaf6d2 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java @@ -37,6 +37,7 @@ import org.apache.cassandra.concurrent.NamedThreadFactory; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.Schema; import org.apache.cassandra.db.*; +import org.apache.cassandra.db.index.SecondaryIndex; import org.apache.cassandra.db.index.SecondaryIndexBuilder; import org.apache.cassandra.dht.Range; import org.apache.cassandra.io.sstable.*; @@ -928,19 +929,13 @@ public class CompactionManager implements CompactionManagerMBean public void runMayThrow() throws InterruptedException, IOException { compactionLock.writeLock().lock(); + try { - for (ColumnFamilyStore cfs : main.concatWithIndexes()) - { - List truncatedSSTables = new ArrayList(); - for (SSTableReader sstable : cfs.getSSTables()) - { - if (!sstable.newSince(truncatedAt)) - truncatedSSTables.add(sstable); - } - if (!truncatedSSTables.isEmpty()) - cfs.markCompacted(truncatedSSTables); - } + main.discardSSTables(truncatedAt); + + for (SecondaryIndex index : main.indexManager.getIndexes()) + index.truncate(truncatedAt); } finally { http://git-wip-us.apache.org/repos/asf/cassandra/blob/54528835/src/java/org/apache/cassandra/db/index/SecondaryIndex.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/index/SecondaryIndex.java b/src/java/org/apache/cassandra/db/index/SecondaryIndex.java index c2d2318..5006217 100644 --- a/src/java/org/apache/cassandra/db/index/SecondaryIndex.java +++ b/src/java/org/apache/cassandra/db/index/SecondaryIndex.java @@ -133,7 +133,13 @@ public abstract class SecondaryIndex * Remove the index and unregisters this index's mbean if one exists */ public abstract void invalidate(); - + + /** + * Truncate all the data from the current index + * + * @param truncatedAt The truncation timestamp, all data before that timestamp should be rejected. + */ + public abstract void truncate(long truncatedAt); /** * Builds the index using the data in the underlying CFS http://git-wip-us.apache.org/repos/asf/cassandra/blob/54528835/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java b/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java index d5fdd2c..77bf954 100644 --- a/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java +++ b/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java @@ -327,7 +327,21 @@ public class SecondaryIndexManager return indexList.keySet(); } - + + /** + * @return all of the secondary indexes without distinction to the (non-)backed by secondary ColumnFamilyStore. + */ + public Collection getIndexes() + { + // we use identity map because per row indexes use same instance across many columns + IdentityHashMap indexList = new IdentityHashMap(); + + for (Map.Entry entry : indexesByColumn.entrySet()) + indexList.put(entry.getValue(), null); + + return indexList.keySet(); + } + /** * @return total current ram size of all indexes */ http://git-wip-us.apache.org/repos/asf/cassandra/blob/54528835/src/java/org/apache/cassandra/db/index/keys/KeysIndex.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/index/keys/KeysIndex.java b/src/java/org/apache/cassandra/db/index/keys/KeysIndex.java index ce26ab3..6ce6242 100644 --- a/src/java/org/apache/cassandra/db/index/keys/KeysIndex.java +++ b/src/java/org/apache/cassandra/db/index/keys/KeysIndex.java @@ -133,6 +133,11 @@ public class KeysIndex extends PerColumnSecondaryIndex indexCfs.invalidate(); } + public void truncate(long truncatedAt) + { + indexCfs.discardSSTables(truncatedAt); + } + public ColumnFamilyStore getIndexCfs() { return indexCfs;