Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 6E60F200B71 for ; Wed, 17 Aug 2016 00:42:09 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 6D00D160ABF; Tue, 16 Aug 2016 22:42:09 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 8CF1E160ABC for ; Wed, 17 Aug 2016 00:42:08 +0200 (CEST) Received: (qmail 98750 invoked by uid 500); 16 Aug 2016 22:42:03 -0000 Mailing-List: contact commits-help@cassandra.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cassandra.apache.org Delivered-To: mailing list commits@cassandra.apache.org Received: (qmail 97854 invoked by uid 99); 16 Aug 2016 22:42:02 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 16 Aug 2016 22:42:02 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id BDA73EF7A4; Tue, 16 Aug 2016 22:42:02 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: mshuler@apache.org To: commits@cassandra.apache.org Date: Tue, 16 Aug 2016 22:42:27 -0000 Message-Id: <7f5e460a54be43fd919fd8171100b6a6@git.apache.org> In-Reply-To: <6f63b6836b6c4f2c9c7a6f6368b33337@git.apache.org> References: <6f63b6836b6c4f2c9c7a6f6368b33337@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [26/50] [abbrv] cassandra git commit: Fix clean interval not sent to commit log for empty memtable flush archived-at: Tue, 16 Aug 2016 22:42:09 -0000 Fix clean interval not sent to commit log for empty memtable flush patch by Branimir Lambov; reviewed by Jake Luciani for CASSANDRA-12436 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ca556c44 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ca556c44 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ca556c44 Branch: refs/heads/cassandra-3.8 Commit: ca556c442049808c013bd20649d6611eb4a01a61 Parents: e60f741 Author: Branimir Lambov Authored: Thu Aug 11 11:17:37 2016 +0300 Committer: T Jake Luciani Committed: Fri Aug 12 11:29:23 2016 -0400 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../apache/cassandra/db/ColumnFamilyStore.java | 22 +++++------ .../cassandra/db/commitlog/CommitLog.java | 5 ++- .../db/commitlog/CommitLogSegment.java | 5 ++- .../db/commitlog/CommitLogCQLTest.java | 41 ++++++++++++++++++++ 5 files changed, 58 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/ca556c44/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index b1b2050..56875ca 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,6 +6,7 @@ * cqlsh: Fix handling of $$-escaped strings (CASSANDRA-12189) * Fix SSL JMX requiring truststore containing server cert (CASSANDRA-12109) Merged from 3.0: + * Fix clean interval not sent to commit log for empty memtable flush (CASSANDRA-12436) * Fix potential resource leak in RMIServerSocketFactoryImpl (CASSANDRA-12331) * Make sure compaction stats are updated when compaction is interrupted (CASSANDRA-12100) * Change commitlog and sstables to track dirty and clean intervals (CASSANDRA-11828) http://git-wip-us.apache.org/repos/asf/cassandra/blob/ca556c44/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 21becfe..aa79e90 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -1072,20 +1072,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean writeBarrier.markBlocking(); writeBarrier.await(); - // mark all memtables as flushing, removing them from the live memtable list, and - // remove any memtables that are already clean from the set we need to flush - Iterator iter = memtables.iterator(); - while (iter.hasNext()) - { - Memtable memtable = iter.next(); + // mark all memtables as flushing, removing them from the live memtable list + for (Memtable memtable : memtables) memtable.cfs.data.markFlushing(memtable); - if (memtable.isClean() || truncate) - { - memtable.cfs.replaceFlushed(memtable, Collections.emptyList()); - reclaim(memtable); - iter.remove(); - } - } metric.memtableSwitchCount.inc(); @@ -1105,6 +1094,13 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean public Collection flushMemtable(Memtable memtable) { + if (memtable.isClean() || truncate) + { + memtable.cfs.replaceFlushed(memtable, Collections.emptyList()); + reclaim(memtable); + return Collections.emptyList(); + } + List> futures = new ArrayList<>(); long totalBytesOnDisk = 0; long maxBytesOnDisk = 0; http://git-wip-us.apache.org/repos/asf/cassandra/blob/ca556c44/src/java/org/apache/cassandra/db/commitlog/CommitLog.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLog.java b/src/java/org/apache/cassandra/db/commitlog/CommitLog.java index 32f69eb..ce92a0a 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLog.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLog.java @@ -321,8 +321,9 @@ public class CommitLog implements CommitLogMBean } else { - logger.trace("Not safe to delete{} commit log segment {}; dirty is {}", - (iter.hasNext() ? "" : " active"), segment, segment.dirtyString()); + if (logger.isTraceEnabled()) + logger.trace("Not safe to delete{} commit log segment {}; dirty is {}", + (iter.hasNext() ? "" : " active"), segment, segment.dirtyString()); } // Don't mark or try to delete any newer segments once we've reached the one containing the http://git-wip-us.apache.org/repos/asf/cassandra/blob/ca556c44/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java index e32c204..94f0ac8 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java @@ -564,7 +564,10 @@ public abstract class CommitLogSegment for (UUID cfId : getDirtyCFIDs()) { CFMetaData m = Schema.instance.getCFMetaData(cfId); - sb.append(m == null ? "" : m.cfName).append(" (").append(cfId).append("), "); + sb.append(m == null ? "" : m.cfName).append(" (").append(cfId) + .append(", dirty: ").append(cfDirty.get(cfId)) + .append(", clean: ").append(cfClean.get(cfId)) + .append("), "); } return sb.toString(); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/ca556c44/test/unit/org/apache/cassandra/db/commitlog/CommitLogCQLTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogCQLTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogCQLTest.java new file mode 100644 index 0000000..7235600 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogCQLTest.java @@ -0,0 +1,41 @@ +package org.apache.cassandra.db.commitlog; + +import java.util.ArrayList; +import java.util.Collection; + +import org.junit.Test; + +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.ColumnFamilyStore; + +public class CommitLogCQLTest extends CQLTester +{ + @Test + public void testTruncateSegmentDiscard() throws Throwable + { + String otherTable = createTable("CREATE TABLE %s (idx INT, data TEXT, PRIMARY KEY(idx));"); + + createTable("CREATE TABLE %s (idx INT, data TEXT, PRIMARY KEY(idx));"); + execute("INSERT INTO %s (idx, data) VALUES (?, ?)", 15, Integer.toString(15)); + flush(); + + // We write something in different table to advance the commit log position. Current table remains clean. + executeFormattedQuery(String.format("INSERT INTO %s.%s (idx, data) VALUES (?, ?)", keyspace(), otherTable), 16, Integer.toString(16)); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + assert cfs.getTracker().getView().getCurrentMemtable().isClean(); + // Calling switchMemtable directly applies Flush even though memtable is empty. This can happen with some races + // (flush with recycling by segment manager). It should still tell commitlog that the memtable's region is clean. + // CASSANDRA-12436 + cfs.switchMemtable(); + + execute("INSERT INTO %s (idx, data) VALUES (?, ?)", 15, Integer.toString(17)); + + Collection active = new ArrayList<>(CommitLog.instance.segmentManager.getActiveSegments()); + CommitLog.instance.forceRecycleAllSegments(); + + // If one of the previous segments remains, it wasn't clean. + active.retainAll(CommitLog.instance.segmentManager.getActiveSegments()); + assert active.isEmpty(); + } +}