cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marc...@apache.org
Subject [1/3] cassandra git commit: Add option to change how often we check for expired sstables with DTCS
Date Tue, 18 Aug 2015 09:02:30 GMT
Repository: cassandra
Updated Branches:
  refs/heads/cassandra-2.2 ee186b6b0 -> 09147a8b6


Add option to change how often we check for expired sstables with DTCS

Patch by marcuse; reviewed by yukim for CASSANDRA-9882


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/17042f19
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/17042f19
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/17042f19

Branch: refs/heads/cassandra-2.2
Commit: 17042f19b084d93bd3bcc35c7282812f456b7640
Parents: f53aacb
Author: Marcus Eriksson <marcuse@apache.org>
Authored: Mon Aug 17 09:50:37 2015 +0200
Committer: Marcus Eriksson <marcuse@apache.org>
Committed: Tue Aug 18 10:47:04 2015 +0200

----------------------------------------------------------------------
 .../DateTieredCompactionStrategy.java           |  7 +++----
 .../DateTieredCompactionStrategyOptions.java    | 21 ++++++++++++++++++++
 .../DateTieredCompactionStrategyTest.java       |  2 +-
 3 files changed, 25 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/17042f19/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index fea4995..2e43352 100644
--- a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@ -38,8 +38,7 @@ public class DateTieredCompactionStrategy extends AbstractCompactionStrategy
 
     protected DateTieredCompactionStrategyOptions options;
     protected volatile int estimatedRemainingTasks;
-    @VisibleForTesting
-    long lastExpiredCheck;
+    private long lastExpiredCheck;
 
     public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map<String, String>
options)
     {
@@ -87,8 +86,8 @@ public class DateTieredCompactionStrategy extends AbstractCompactionStrategy
         Set<SSTableReader> uncompacting = cfs.getUncompactingSSTables();
 
         Set<SSTableReader> expired = Collections.emptySet();
-        // we only check for expired sstables every 10 minutes due to it being an expensive
operation
-        if (System.currentTimeMillis() - lastExpiredCheck > TimeUnit.MINUTES.toMillis(10))
+        // we only check for expired sstables every 10 minutes (by default) due to it being
an expensive operation
+        if (System.currentTimeMillis() - lastExpiredCheck > options.expiredSSTableCheckFrequency)
         {
             // Find fully expired SSTables. Those will be included no matter what.
             expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, cfs.getOverlappingSSTables(uncompacting),
gcBefore);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/17042f19/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
index f54c020..0cbf90e 100644
--- a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
+++ b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
@@ -27,12 +27,15 @@ public final class DateTieredCompactionStrategyOptions
     protected static final TimeUnit DEFAULT_TIMESTAMP_RESOLUTION = TimeUnit.MICROSECONDS;
     protected static final double DEFAULT_MAX_SSTABLE_AGE_DAYS = 365;
     protected static final long DEFAULT_BASE_TIME_SECONDS = 60;
+    protected static final int DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS = 60 * 10;
     protected static final String TIMESTAMP_RESOLUTION_KEY = "timestamp_resolution";
     protected static final String MAX_SSTABLE_AGE_KEY = "max_sstable_age_days";
     protected static final String BASE_TIME_KEY = "base_time_seconds";
+    protected static final String EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY = "expired_sstable_check_frequency_seconds";
 
     protected final long maxSSTableAge;
     protected final long baseTime;
+    protected final long expiredSSTableCheckFrequency;
 
     public DateTieredCompactionStrategyOptions(Map<String, String> options)
     {
@@ -43,12 +46,15 @@ public final class DateTieredCompactionStrategyOptions
         maxSSTableAge = Math.round(fractionalDays * timestampResolution.convert(1, TimeUnit.DAYS));
         optionValue = options.get(BASE_TIME_KEY);
         baseTime = timestampResolution.convert(optionValue == null ? DEFAULT_BASE_TIME_SECONDS
: Long.parseLong(optionValue), TimeUnit.SECONDS);
+        optionValue = options.get(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY);
+        expiredSSTableCheckFrequency = TimeUnit.MILLISECONDS.convert(optionValue == null
? DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS : Long.parseLong(optionValue), TimeUnit.SECONDS);
     }
 
     public DateTieredCompactionStrategyOptions()
     {
         maxSSTableAge = Math.round(DEFAULT_MAX_SSTABLE_AGE_DAYS * DEFAULT_TIMESTAMP_RESOLUTION.convert(1,
TimeUnit.DAYS));
         baseTime = DEFAULT_TIMESTAMP_RESOLUTION.convert(DEFAULT_BASE_TIME_SECONDS, TimeUnit.SECONDS);
+        expiredSSTableCheckFrequency = TimeUnit.MILLISECONDS.convert(DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS,
TimeUnit.SECONDS);
     }
 
     public static Map<String, String> validateOptions(Map<String, String> options,
Map<String, String> uncheckedOptions) throws  ConfigurationException
@@ -92,9 +98,24 @@ public final class DateTieredCompactionStrategyOptions
             throw new ConfigurationException(String.format("%s is not a parsable int (base10)
for %s", optionValue, BASE_TIME_KEY), e);
         }
 
+        optionValue = options.get(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY);
+        try
+        {
+            long expiredCheckFrequency = optionValue == null ? DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS
: Long.parseLong(optionValue);
+            if (expiredCheckFrequency < 0)
+            {
+                throw new ConfigurationException(String.format("%s must not be negative,
but was %d", EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY, expiredCheckFrequency));
+            }
+        }
+        catch (NumberFormatException e)
+        {
+            throw new ConfigurationException(String.format("%s is not a parsable int (base10)
for %s", optionValue, EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY), e);
+        }
+
         uncheckedOptions.remove(MAX_SSTABLE_AGE_KEY);
         uncheckedOptions.remove(BASE_TIME_KEY);
         uncheckedOptions.remove(TIMESTAMP_RESOLUTION_KEY);
+        uncheckedOptions.remove(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY);
 
         return uncheckedOptions;
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/17042f19/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
index 0084a16..6d9bfc2 100644
--- a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
@@ -305,10 +305,10 @@ public class DateTieredCompactionStrategyTest extends SchemaLoader
         options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "30");
         options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY, "MILLISECONDS");
         options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY, Double.toString((1d
/ (24 * 60 * 60))));
+        options.put(DateTieredCompactionStrategyOptions.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY,
"0");
         DateTieredCompactionStrategy dtcs = new DateTieredCompactionStrategy(cfs, options);
         dtcs.startup();
         assertNull(dtcs.getNextBackgroundTask((int) (System.currentTimeMillis() / 1000)));
-        dtcs.lastExpiredCheck = 0;
         Thread.sleep(2000);
         AbstractCompactionTask t = dtcs.getNextBackgroundTask((int) (System.currentTimeMillis()/1000));
         assertNotNull(t);


Mime
View raw message