cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marc...@apache.org
Subject [07/10] cassandra git commit: Merge branch 'cassandra-2.2' into cassandra-3.0
Date Wed, 16 Mar 2016 08:37:57 GMT
Merge branch 'cassandra-2.2' into cassandra-3.0


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b623e82c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b623e82c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b623e82c

Branch: refs/heads/trunk
Commit: b623e82c4527aeb15f12ae975adb5d3cf864b34d
Parents: a479fb0 a48b836
Author: Marcus Eriksson <marcuse@apache.org>
Authored: Wed Mar 16 09:22:44 2016 +0100
Committer: Marcus Eriksson <marcuse@apache.org>
Committed: Wed Mar 16 09:23:04 2016 +0100

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../writers/MaxSSTableSizeWriter.java           | 27 ++++++++++++--------
 2 files changed, 18 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/b623e82c/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 3b5e51f,a69164e..13c682f
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,12 -1,8 +1,13 @@@
 -2.2.6
 +3.0.5
 + * Fix sstabledump to include missing info in debug output (CASSANDRA-11321)
 + * Establish and implement canonical bulk reading workload(s) (CASSANDRA-10331)
 + * Fix paging for IN queries on tables without clustering columns (CASSANDRA-11208)
 + * Remove recursive call from CompositesSearcher (CASSANDRA-11304)
 + * Fix filtering on non-primary key columns for queries without index (CASSANDRA-6377)
 + * Fix sstableloader fail when using materialized view (CASSANDRA-11275)
 +Merged from 2.2:
+  * Fix bloom filter sizing with LCS (CASSANDRA-11344)
   * (cqlsh) Fix error when result is 0 rows with EXPAND ON (CASSANDRA-11092)
 - * Fix intra-node serialization issue for multicolumn-restrictions (CASSANDRA-11196)
 - * Non-obsoleting compaction operations over compressed files can impose rate limit on normal
reads (CASSANDRA-11301)
   * Add missing newline at end of bin/cqlsh (CASSANDRA-11325)
   * Fix AE in nodetool cfstats (backport CASSANDRA-10859) (CASSANDRA-11297)
   * Unresolved hostname leads to replace being ignored (CASSANDRA-11210)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/b623e82c/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
index 142fe87,2bae3b8..b206498
--- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
@@@ -20,10 -20,10 +20,11 @@@ package org.apache.cassandra.db.compact
  import java.util.Set;
  
  import org.apache.cassandra.db.ColumnFamilyStore;
 +import org.apache.cassandra.db.Directories;
  import org.apache.cassandra.db.RowIndexEntry;
 -import org.apache.cassandra.db.compaction.AbstractCompactedRow;
 +import org.apache.cassandra.db.SerializationHeader;
+ import org.apache.cassandra.db.compaction.OperationType;
 +import org.apache.cassandra.db.rows.UnfilteredRowIterator;
  import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
  import org.apache.cassandra.io.sstable.Descriptor;
  import org.apache.cassandra.io.sstable.format.SSTableReader;
@@@ -63,39 -45,50 +63,46 @@@ public class MaxSSTableSizeWriter exten
          this.allSSTables = txn.originals();
          this.level = level;
          this.maxSSTableSize = maxSSTableSize;
-         long totalSize = cfs.getExpectedCompactedFileSize(nonExpiredSSTables, txn.opType());
 -        long totalSize = getTotalWriteSize(nonExpiredSSTables, estimatedTotalKeys, cfs,
compactionType);
++
++        long totalSize = getTotalWriteSize(nonExpiredSSTables, estimatedTotalKeys, cfs,
txn.opType());
          expectedWriteSize = Math.min(maxSSTableSize, totalSize);
-         estimatedTotalKeys = SSTableReader.getApproximateKeyCount(nonExpiredSSTables);
-         estimatedSSTables = Math.max(1, estimatedTotalKeys / maxSSTableSize);
+         estimatedSSTables = Math.max(1, totalSize / maxSSTableSize);
 -        File sstableDirectory = cfs.directories.getLocationForDisk(getWriteDirectory(expectedWriteSize));
 -        @SuppressWarnings("resource")
 -        SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(sstableDirectory)),
 -                                                    estimatedTotalKeys / estimatedSSTables,
 -                                                    minRepairedAt,
 -                                                    cfs.metadata,
 -                                                    cfs.partitioner,
 -                                                    new MetadataCollector(allSSTables, cfs.metadata.comparator,
level));
 -        sstableWriter.switchWriter(writer);
+     }
+ 
+     /**
+      * Gets the estimated total amount of data to write during compaction
+      */
+     private static long getTotalWriteSize(Iterable<SSTableReader> nonExpiredSSTables,
long estimatedTotalKeys, ColumnFamilyStore cfs, OperationType compactionType)
+     {
+         long estimatedKeysBeforeCompaction = 0;
+         for (SSTableReader sstable : nonExpiredSSTables)
+             estimatedKeysBeforeCompaction += sstable.estimatedKeys();
+         estimatedKeysBeforeCompaction = Math.max(1, estimatedKeysBeforeCompaction);
+         double estimatedCompactionRatio = (double) estimatedTotalKeys / estimatedKeysBeforeCompaction;
+         return Math.round(estimatedCompactionRatio * cfs.getExpectedCompactedFileSize(nonExpiredSSTables,
compactionType));
      }
  
      @Override
 -    public boolean append(AbstractCompactedRow row)
 +    public boolean realAppend(UnfilteredRowIterator partition)
      {
 -        RowIndexEntry rie = sstableWriter.append(row);
 +        RowIndexEntry rie = sstableWriter.append(partition);
          if (sstableWriter.currentWriter().getOnDiskFilePointer() > maxSSTableSize)
 -        {
 -            File sstableDirectory = cfs.directories.getLocationForDisk(getWriteDirectory(expectedWriteSize));
 -            @SuppressWarnings("resource")
 -            SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getTempSSTablePath(sstableDirectory)),
 -                                                                estimatedTotalKeys / estimatedSSTables,
 -                                                                minRepairedAt,
 -                                                                cfs.metadata,
 -                                                                cfs.partitioner,
 -                                                                new MetadataCollector(allSSTables,
cfs.metadata.comparator, level));
 -
 -            sstableWriter.switchWriter(writer);
 -        }
 +            switchCompactionLocation(getWriteDirectory(expectedWriteSize));
          return rie != null;
      }
 +
 +    public void switchCompactionLocation(Directories.DataDirectory location)
 +    {
 +        @SuppressWarnings("resource")
 +        SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(getDirectories().getLocationForDisk(location))),
 +                                                    estimatedTotalKeys / estimatedSSTables,
 +                                                    minRepairedAt,
 +                                                    cfs.metadata,
 +                                                    new MetadataCollector(allSSTables, cfs.metadata.comparator,
level),
 +                                                    SerializationHeader.make(cfs.metadata,
nonExpiredSSTables),
 +                                                    txn);
 +
 +        sstableWriter.switchWriter(writer);
 +
 +    }
- 
-     @Override
-     public long estimatedKeys()
-     {
-         return estimatedTotalKeys;
-     }
  }


Mime
View raw message