cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marc...@apache.org
Subject [07/10] cassandra git commit: Merge branch 'cassandra-2.2' into cassandra-3.0
Date Tue, 05 Sep 2017 08:47:26 GMT
Merge branch 'cassandra-2.2' into cassandra-3.0


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/fb92a526
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/fb92a526
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/fb92a526

Branch: refs/heads/trunk
Commit: fb92a52624c13969386760b967b7168a87e857bb
Parents: f4014bb 4e834c5
Author: Marcus Eriksson <marcuse@apache.org>
Authored: Tue Sep 5 10:38:39 2017 +0200
Committer: Marcus Eriksson <marcuse@apache.org>
Committed: Tue Sep 5 10:38:39 2017 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../io/sstable/format/SSTableReader.java        |   8 +-
 .../sstable/IndexSummaryRedistributionTest.java | 142 +++++++++++++++++++
 3 files changed, 147 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/fb92a526/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 870351c,1abd7de..1ed3fe6
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,39 -1,9 +1,40 @@@
 -2.2.11
 +3.0.15
 + * Fix MV timestamp issues (CASSANDRA-11500)
 + * Better tolerate improperly formatted bcrypt hashes (CASSANDRA-13626) 
 + * Fix race condition in read command serialization (CASSANDRA-13363)
 + * Enable segement creation before recovering commitlogs (CASSANDRA-13587)
 + * Fix AssertionError in short read protection (CASSANDRA-13747)
 + * Don't skip corrupted sstables on startup (CASSANDRA-13620)
 + * Fix the merging of cells with different user type versions (CASSANDRA-13776)
 + * Copy session properties on cqlsh.py do_login (CASSANDRA-13640)
 + * Potential AssertionError during ReadRepair of range tombstone and partition deletions
(CASSANDRA-13719)
 + * Don't let stress write warmup data if n=0 (CASSANDRA-13773)
 + * Gossip thread slows down when using batch commit log (CASSANDRA-12966)
 + * Randomize batchlog endpoint selection with only 1 or 2 racks (CASSANDRA-12884)
 + * Fix digest calculation for counter cells (CASSANDRA-13750)
 + * Fix ColumnDefinition.cellValueType() for non-frozen collection and change SSTabledump
to use type.toJSONString() (CASSANDRA-13573)
 + * Skip materialized view addition if the base table doesn't exist (CASSANDRA-13737)
 + * Drop table should remove corresponding entries in dropped_columns table (CASSANDRA-13730)
 + * Log warn message until legacy auth tables have been migrated (CASSANDRA-13371)
 + * Fix incorrect [2.1 <- 3.0] serialization of counter cells created in 2.0 (CASSANDRA-13691)
 + * Fix invalid writetime for null cells (CASSANDRA-13711)
 + * Fix ALTER TABLE statement to atomically propagate changes to the table and its MVs (CASSANDRA-12952)
 + * Fixed ambiguous output of nodetool tablestats command (CASSANDRA-13722)
 + * JMXEnabledThreadPoolExecutor with corePoolSize equal to maxPoolSize (Backport CASSANDRA-13329)
 + * Fix Digest mismatch Exception if hints file has UnknownColumnFamily (CASSANDRA-13696)
 + * Purge tombstones created by expired cells (CASSANDRA-13643)
 + * Make concat work with iterators that have different subsets of columns (CASSANDRA-13482)
 + * Set test.runners based on cores and memory size (CASSANDRA-13078)
 + * Allow different NUMACTL_ARGS to be passed in (CASSANDRA-13557)
 + * Allow native function calls in CQLSSTableWriter (CASSANDRA-12606)
 + * Fix secondary index queries on COMPACT tables (CASSANDRA-13627)
 + * Nodetool listsnapshots output is missing a newline, if there are no snapshots (CASSANDRA-13568)
 + * sstabledump reports incorrect usage for argument order (CASSANDRA-13532)
 +Merged from 2.2:
+  * Fix load over calculated issue in IndexSummaryRedistribution (CASSANDRA-13738)
   * Fix compaction and flush exception not captured (CASSANDRA-13833)
 - * Make BatchlogManagerMBean.forceBatchlogReplay() blocking (CASSANDRA-13809)
   * Uncaught exceptions in Netty pipeline (CASSANDRA-13649)
 - * Prevent integer overflow on exabyte filesystems (CASSANDRA-13067) 
 + * Prevent integer overflow on exabyte filesystems (CASSANDRA-13067)
   * Fix queries with LIMIT and filtering on clustering columns (CASSANDRA-11223)
   * Fix potential NPE when resume bootstrap fails (CASSANDRA-13272)
   * Fix toJSONString for the UDT, tuple and collection types (CASSANDRA-13592)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/fb92a526/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index d56b3e7,6666885..25ba212
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@@ -1123,10 -1169,14 +1123,10 @@@ public abstract class SSTableReader ext
                  saveSummary(ibuilder, dbuilder, newSummary);
              }
  
-             long newSize = bytesOnDisk();
-             StorageMetrics.load.inc(newSize - oldSize);
-             parent.metric.liveDiskSpaceUsed.inc(newSize - oldSize);
-             parent.metric.totalDiskSpaceUsed.inc(newSize - oldSize);
+             // The new size will be added in Transactional.commit() as an updated SSTable,
more details: CASSANDRA-13738
+             StorageMetrics.load.dec(oldSize);
+             parent.metric.liveDiskSpaceUsed.dec(oldSize);
++            parent.metric.totalDiskSpaceUsed.dec(oldSize);
  
              return cloneAndReplace(first, OpenReason.METADATA_CHANGE, newSummary);
          }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/fb92a526/test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java
index 0000000,77fd69a..31a57e1
mode 000000,100644..100644
--- a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java
@@@ -1,0 -1,145 +1,142 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ package org.apache.cassandra.io.sstable;
+ 
+ import java.io.IOException;
+ import java.nio.ByteBuffer;
+ import java.util.ArrayList;
+ import java.util.List;
+ import java.util.concurrent.ExecutionException;
+ import java.util.concurrent.Future;
+ 
+ import org.junit.BeforeClass;
+ import org.junit.Test;
+ 
+ import org.apache.cassandra.SchemaLoader;
 -import org.apache.cassandra.Util;
 -import org.apache.cassandra.cache.CachingOptions;
 -import org.apache.cassandra.config.KSMetaData;
+ import org.apache.cassandra.db.ColumnFamilyStore;
 -import org.apache.cassandra.db.DecoratedKey;
+ import org.apache.cassandra.db.Keyspace;
 -import org.apache.cassandra.db.Mutation;
++import org.apache.cassandra.db.RowUpdateBuilder;
+ import org.apache.cassandra.exceptions.ConfigurationException;
+ import org.apache.cassandra.io.sstable.format.SSTableReader;
 -import org.apache.cassandra.locator.SimpleStrategy;
+ import org.apache.cassandra.metrics.RestorableMeter;
+ import org.apache.cassandra.metrics.StorageMetrics;
++import org.apache.cassandra.schema.CachingParams;
++import org.apache.cassandra.schema.KeyspaceParams;
+ 
+ import static org.junit.Assert.assertEquals;
+ 
+ public class IndexSummaryRedistributionTest
+ {
+     private static final String KEYSPACE1 = "IndexSummaryRedistributionTest";
+     private static final String CF_STANDARD = "Standard";
+ 
+     @BeforeClass
+     public static void defineSchema() throws ConfigurationException
+     {
+         SchemaLoader.prepareServer();
+         SchemaLoader.createKeyspace(KEYSPACE1,
 -                                    SimpleStrategy.class,
 -                                    KSMetaData.optsWithRF(1),
++                                    KeyspaceParams.simple(1),
+                                     SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD)
+                                                 .minIndexInterval(8)
+                                                 .maxIndexInterval(256)
 -                                                .caching(CachingOptions.NONE));
++                                                .caching(CachingParams.CACHE_NOTHING));
+     }
+ 
+     @Test
+     public void testMetricsLoadAfterRedistribution() throws IOException
+     {
+         String ksname = KEYSPACE1;
+         String cfname = CF_STANDARD;
+         Keyspace keyspace = Keyspace.open(ksname);
+         ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+         int numSSTables = 1;
+         int numRows = 1024 * 10;
+         long load = StorageMetrics.load.getCount();
+         StorageMetrics.load.dec(load); // reset the load metric
+         createSSTables(ksname, cfname, numSSTables, numRows);
+ 
 -        List<SSTableReader> sstables = new ArrayList<>(cfs.getSSTables());
++        List<SSTableReader> sstables = new ArrayList<>(cfs.getLiveSSTables());
+         for (SSTableReader sstable : sstables)
+             sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0));
+ 
+         long oldSize = 0;
+         for (SSTableReader sstable : sstables)
+         {
 -            assertEquals(cfs.metadata.getMinIndexInterval(), sstable.getEffectiveIndexInterval(),
0.001);
++            assertEquals(cfs.metadata.params.minIndexInterval, sstable.getEffectiveIndexInterval(),
0.001);
+             oldSize += sstable.bytesOnDisk();
+         }
+ 
+         load = StorageMetrics.load.getCount();
 -
+         long others = load - oldSize; // Other SSTables size, e.g. schema and other system
SSTables
+ 
 -        int originalMinIndexInterval = cfs.metadata.getMinIndexInterval();
++        int originalMinIndexInterval = cfs.metadata.params.minIndexInterval;
+         // double the min_index_interval
+         cfs.metadata.minIndexInterval(originalMinIndexInterval * 2);
+         IndexSummaryManager.instance.redistributeSummaries();
+ 
+         long newSize = 0;
 -        for (SSTableReader sstable : cfs.getSSTables())
++        for (SSTableReader sstable : cfs.getLiveSSTables())
+         {
 -            assertEquals(cfs.metadata.getMinIndexInterval(), sstable.getEffectiveIndexInterval(),
0.001);
 -            assertEquals(numRows / cfs.metadata.getMinIndexInterval(), sstable.getIndexSummarySize());
++            assertEquals(cfs.metadata.params.minIndexInterval, sstable.getEffectiveIndexInterval(),
0.001);
++            assertEquals(numRows / cfs.metadata.params.minIndexInterval, sstable.getIndexSummarySize());
+             newSize += sstable.bytesOnDisk();
+         }
+         newSize += others;
+         load = StorageMetrics.load.getCount();
+ 
+         // new size we calculate should be almost the same as the load in metrics
+         assertEquals(newSize, load, newSize / 10);
+     }
+ 
+     private void createSSTables(String ksname, String cfname, int numSSTables, int numRows)
+     {
+         Keyspace keyspace = Keyspace.open(ksname);
+         ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+         cfs.truncateBlocking();
+         cfs.disableAutoCompaction();
+ 
+         ArrayList<Future> futures = new ArrayList<>(numSSTables);
+         ByteBuffer value = ByteBuffer.wrap(new byte[100]);
+         for (int sstable = 0; sstable < numSSTables; sstable++)
+         {
+             for (int row = 0; row < numRows; row++)
+             {
 -                DecoratedKey key = Util.dk(String.format("%3d", row));
 -                Mutation rm = new Mutation(ksname, key.getKey());
 -                rm.add(cfname, Util.cellname("column"), value, 0);
 -                rm.applyUnsafe();
++                String key = String.format("%3d", row);
++                new RowUpdateBuilder(cfs.metadata, 0, key)
++                .clustering("column")
++                .add("val", value)
++                .build()
++                .applyUnsafe();
+             }
+             futures.add(cfs.forceFlush());
+         }
+         for (Future future : futures)
+         {
+             try
+             {
+                 future.get();
+             }
+             catch (InterruptedException | ExecutionException e)
+             {
+                 throw new RuntimeException(e);
+             }
+         }
 -        assertEquals(numSSTables, cfs.getSSTables().size());
++        assertEquals(numSSTables, cfs.getLiveSSTables().size());
+     }
+ }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


Mime
View raw message