cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marc...@apache.org
Subject [2/5] cassandra git commit: Merge branch 'cassandra-2.0' into cassandra-2.1
Date Mon, 17 Aug 2015 06:56:37 GMT
Merge branch 'cassandra-2.0' into cassandra-2.1

Conflicts:
	CHANGES.txt
	src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/26ff1507
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/26ff1507
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/26ff1507

Branch: refs/heads/trunk
Commit: 26ff15070eb67779050975a9ae07053f4b5e5b04
Parents: 7a6a509 f53aacb
Author: Marcus Eriksson <marcuse@apache.org>
Authored: Mon Aug 17 08:30:17 2015 +0200
Committer: Marcus Eriksson <marcuse@apache.org>
Committed: Mon Aug 17 08:30:17 2015 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  | 55 +++++++++++++++++---
 .../DateTieredCompactionStrategy.java           | 13 ++++-
 .../DateTieredCompactionStrategyTest.java       |  1 +
 4 files changed, 60 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/26ff1507/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 5a76978,905445e..4626899
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,27 -1,5 +1,28 @@@
 -2.0.17
 +2.1.9
 + * (cqlsh) Allow encoding to be set through command line (CASSANDRA-10004)
 + * Add new JMX methods to change local compaction strategy (CASSANDRA-9965)
 + * Write hints for paxos commits (CASSANDRA-7342)
 + * (cqlsh) Fix timestamps before 1970 on Windows, always
 +   use UTC for timestamp display (CASSANDRA-10000)
 + * (cqlsh) Avoid overwriting new config file with old config
 +   when both exist (CASSANDRA-9777)
 + * Release snapshot selfRef when doing snapshot repair (CASSANDRA-9998)
 + * Cannot replace token does not exist - DN node removed as Fat Client (CASSANDRA-9871)
 + * Fix handling of enable/disable autocompaction (CASSANDRA-9899)
 + * Commit log segment recycling is disabled by default (CASSANDRA-9896)
 + * Add consistency level to tracing ouput (CASSANDRA-9827)
 + * Fix MarshalException when upgrading superColumn family (CASSANDRA-9582)
 + * Fix broken logging for "empty" flushes in Memtable (CASSANDRA-9837)
 + * Handle corrupt files on startup (CASSANDRA-9686)
 + * Fix clientutil jar and tests (CASSANDRA-9760)
 + * (cqlsh) Allow the SSL protocol version to be specified through the
 +   config file or environment variables (CASSANDRA-9544)
 + * Remove repair snapshot leftover on startup (CASSANDRA-7357)
 + * Use random nodes for batch log when only 2 racks (CASSANDRA-8735)
 + * Ensure atomicity inside thrift and stream session (CASSANDRA-7757)
 + * Fix nodetool info error when the node is not joined (CASSANDRA-9031)
 +Merged from 2.0:
+  * Make getFullyExpiredSSTables less expensive (CASSANDRA-9882)
   * Add tool to find why expired sstables are not getting dropped (CASSANDRA-10015)
   * Remove erroneous pending HH tasks from tpstats/jmx (CASSANDRA-9129)
   * Don't cast expected bf size to an int (CASSANDRA-9959)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/26ff1507/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 7364528,eb688f7..25b3e57
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -1367,20 -1025,59 +1367,59 @@@ public class ColumnFamilyStore implemen
  
          // a normal compaction won't ever have an empty sstables list, but we create a skeleton
          // compaction controller for streaming, and that passes an empty list.
 -        if (sstables.isEmpty())
 +        if (!sstables.iterator().hasNext())
              return ImmutableSet.of();
  
-         DataTracker.SSTableIntervalTree tree = data.getView().intervalTree;
- 
-         Set<SSTableReader> results = null;
-         for (SSTableReader sstable : sstables)
 -        List<SSTableReader> sortedByFirst = new ArrayList<>(sstables);
++        List<SSTableReader> sortedByFirst = Lists.newArrayList(sstables);
+         Collections.sort(sortedByFirst, new Comparator<SSTableReader>()
+         {
+             @Override
+             public int compare(SSTableReader o1, SSTableReader o2)
+             {
+                 return o1.first.compareTo(o2.first);
+             }
+         });
+         List<Interval<RowPosition, SSTableReader>> intervals = new ArrayList<>();
+         DecoratedKey first = null, last = null;
+         /*
+         normalize the intervals covered by the sstables
+         assume we have sstables like this (brackets representing first/last key in the sstable);
+         [   ] [   ]    [   ]   [  ]
+            [   ]         [       ]
+         then we can, instead of searching the interval tree 6 times, normalize the intervals
and
+         only query the tree 2 times, for these intervals;
+         [         ]    [          ]
+          */
+         for (SSTableReader sstable : sortedByFirst)
          {
-             Set<SSTableReader> overlaps = ImmutableSet.copyOf(tree.search(Interval.<RowPosition,
SSTableReader>create(sstable.first, sstable.last)));
-             results = results == null ? overlaps : Sets.union(results, overlaps).immutableCopy();
+             if (first == null)
+             {
+                 first = sstable.first;
+                 last = sstable.last;
+             }
+             else
+             {
+                 if (sstable.first.compareTo(last) <= 0) // we do overlap
+                 {
+                     if (sstable.last.compareTo(last) > 0)
+                         last = sstable.last;
+                 }
+                 else
+                 {
+                     intervals.add(Interval.<RowPosition, SSTableReader>create(first,
last));
+                     first = sstable.first;
+                     last = sstable.last;
+                 }
+             }
          }
-         results = Sets.difference(results, ImmutableSet.copyOf(sstables));
+         intervals.add(Interval.<RowPosition, SSTableReader>create(first, last));
+         DataTracker.SSTableIntervalTree tree = data.getView().intervalTree;
+         Set<SSTableReader> results = new HashSet<>();
+ 
+         for (Interval<RowPosition, SSTableReader> interval : intervals)
+             results.addAll(tree.search(interval));
  
-         return results;
+         return Sets.difference(results, ImmutableSet.copyOf(sstables));
      }
  
      /**

http://git-wip-us.apache.org/repos/asf/cassandra/blob/26ff1507/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index dec0cef,fea4995..41c304b
--- a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@@ -35,9 -36,10 +36,11 @@@ public class DateTieredCompactionStrate
  {
      private static final Logger logger = LoggerFactory.getLogger(DateTieredCompactionStrategy.class);
  
 -    protected DateTieredCompactionStrategyOptions options;
 +    private final DateTieredCompactionStrategyOptions options;
      protected volatile int estimatedRemainingTasks;
 +    private final Set<SSTableReader> sstables = new HashSet<>();
+     @VisibleForTesting
+     long lastExpiredCheck;
  
      public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map<String, String>
options)
      {
@@@ -76,13 -81,19 +79,19 @@@
       */
      private List<SSTableReader> getNextBackgroundSSTables(final int gcBefore)
      {
 -        if (!isEnabled() || cfs.getSSTables().isEmpty())
 +        if (cfs.getSSTables().isEmpty())
              return Collections.emptyList();
  
 -        Set<SSTableReader> uncompacting = cfs.getUncompactingSSTables();
 +        Set<SSTableReader> uncompacting = Sets.intersection(sstables, cfs.getUncompactingSSTables());
  
-         // Find fully expired SSTables. Those will be included no matter what.
-         Set<SSTableReader> expired = CompactionController.getFullyExpiredSSTables(cfs,
uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore);
+         Set<SSTableReader> expired = Collections.emptySet();
+         // we only check for expired sstables every 10 minutes due to it being an expensive
operation
+         if (System.currentTimeMillis() - lastExpiredCheck > TimeUnit.MINUTES.toMillis(10))
+         {
+             // Find fully expired SSTables. Those will be included no matter what.
+             expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, cfs.getOverlappingSSTables(uncompacting),
gcBefore);
+             lastExpiredCheck = System.currentTimeMillis();
+         }
          Set<SSTableReader> candidates = Sets.newHashSet(filterSuspectSSTables(uncompacting));
  
          List<SSTableReader> compactionCandidates = new ArrayList<>(getNextNonExpiredSSTables(Sets.difference(candidates,
expired), gcBefore));

http://git-wip-us.apache.org/repos/asf/cassandra/blob/26ff1507/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
index 14e22f0,0084a16..cea835f
--- a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
@@@ -306,10 -306,9 +306,11 @@@ public class DateTieredCompactionStrate
          options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY, "MILLISECONDS");
          options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY, Double.toString((1d
/ (24 * 60 * 60))));
          DateTieredCompactionStrategy dtcs = new DateTieredCompactionStrategy(cfs, options);
 +        for (SSTableReader sstable : cfs.getSSTables())
 +            dtcs.addSSTable(sstable);
          dtcs.startup();
          assertNull(dtcs.getNextBackgroundTask((int) (System.currentTimeMillis() / 1000)));
+         dtcs.lastExpiredCheck = 0;
          Thread.sleep(2000);
          AbstractCompactionTask t = dtcs.getNextBackgroundTask((int) (System.currentTimeMillis()/1000));
          assertNotNull(t);


Mime
View raw message