cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From stefa...@apache.org
Subject [09/10] cassandra git commit: Merge branch 'cassandra-3.0' into cassandra-3.7
Date Thu, 26 May 2016 02:57:26 GMT
Merge branch 'cassandra-3.0' into cassandra-3.7


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/68d20edb
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/68d20edb
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/68d20edb

Branch: refs/heads/cassandra-3.7
Commit: 68d20edb2ba89349303dee4ae3be75f8d363c3ff
Parents: f39a352 85cc390
Author: Stefania Alborghetti <stefania.alborghetti@datastax.com>
Authored: Thu May 26 10:51:46 2016 +0800
Committer: Stefania Alborghetti <stefania.alborghetti@datastax.com>
Committed: Thu May 26 10:52:31 2016 +0800

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 NEWS.txt                                        |  11 +
 conf/cassandra.yaml                             |   8 +-
 .../org/apache/cassandra/config/Config.java     |   6 +
 .../cassandra/config/DatabaseDescriptor.java    |   8 +
 .../apache/cassandra/db/ClusteringPrefix.java   |   4 +-
 .../columniterator/AbstractSSTableIterator.java |   2 +-
 .../cassandra/db/marshal/AbstractType.java      |  46 +++-
 src/java/org/apache/cassandra/db/rows/Rows.java |   2 +-
 .../cassandra/db/rows/UnfilteredSerializer.java |  13 +-
 .../io/sstable/SSTableIdentityIterator.java     |   5 +
 .../io/sstable/format/big/BigTableScanner.java  |   2 +-
 .../io/util/RebufferingInputStream.java         |   2 +-
 .../compaction/BlacklistingCompactionsTest.java |  45 +++-
 .../cassandra/index/sasi/SASIIndexTest.java     |   8 +-
 .../sstable/SSTableCorruptionDetectionTest.java | 247 +++++++++++++++++++
 .../cassandra/io/sstable/SSTableWriterTest.java |  49 +++-
 .../io/sstable/SSTableWriterTestBase.java       |  10 +-
 18 files changed, 439 insertions(+), 30 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index d77aa1e,ddfb24f..b2d31bb
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,6 -1,5 +1,7 @@@
 -3.0.7
 +3.7
 + * Don't use static dataDirectories field in Directories instances (CASSANDRA-11647)
 +Merged from 3.0:
+  * Prevent OOM failures on SSTable corruption, improve tests for corruption detection (CASSANDRA-9530)
   * Use CFS.initialDirectories when clearing snapshots (CASSANDRA-11705)
   * Allow compaction strategies to disable early open (CASSANDRA-11754)
   * Refactor Materialized View code (CASSANDRA-11475)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/NEWS.txt
----------------------------------------------------------------------
diff --cc NEWS.txt
index ac33a60,ac1ef17..2f5e3b5
--- a/NEWS.txt
+++ b/NEWS.txt
@@@ -13,7 -13,18 +13,18 @@@ restore snapshots created with the prev
  'sstableloader' tool. You can upgrade the file format of your snapshots
  using the provided 'sstableupgrade' tool.
  
 -3.0.7
 -=====
++3.7
++===
+ 
+ Upgrading
+ ---------
+    - A maximum size for SSTables values has been introduced, to prevent out of memory
+      exceptions when reading corrupt SSTables. This maximum size can be set via
+      max_value_size_in_mb in cassandra.yaml. The default is 256MB, which matches the default
+      value of native_transport_max_frame_size_in_mb. SSTables will be considered corrupt
if
+      they contain values whose size exceeds this limit. See CASSANDRA-9530 for more details.
+ 
 -3.0.6
 +3.6
  =====
  
  New features

http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/conf/cassandra.yaml
----------------------------------------------------------------------
diff --cc conf/cassandra.yaml
index 959d855,4b92f64..dcd5278
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@@ -979,65 -946,7 +980,70 @@@ enable_scripted_user_defined_functions
  # setting.
  windows_timer_interval: 1
  
 +
 +# Enables encrypting data at-rest (on disk). Different key providers can be plugged in,
but the default reads from
 +# a JCE-style keystore. A single keystore can hold multiple keys, but the one referenced
by
 +# the "key_alias" is the only key that will be used for encrypt opertaions; previously used
keys
 +# can still (and should!) be in the keystore and will be used on decrypt operations
 +# (to handle the case of key rotation).
 +#
 +# It is strongly recommended to download and install Java Cryptography Extension (JCE)
 +# Unlimited Strength Jurisdiction Policy Files for your version of the JDK.
 +# (current link: http://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html)
 +#
 +# Currently, only the following file types are supported for transparent data encryption,
although
 +# more are coming in future cassandra releases: commitlog, hints
 +transparent_data_encryption_options:
 +    enabled: false
 +    chunk_length_kb: 64
 +    cipher: AES/CBC/PKCS5Padding
 +    key_alias: testing:1
 +    # CBC IV length for AES needs to be 16 bytes (which is also the default size)
 +    # iv_length: 16
 +    key_provider: 
 +      - class_name: org.apache.cassandra.security.JKSKeyProvider
 +        parameters: 
 +          - keystore: conf/.keystore
 +            keystore_password: cassandra
 +            store_type: JCEKS
 +            key_password: cassandra
 +
 +
 +#####################
 +# SAFETY THRESHOLDS #
 +#####################
 +
 +# When executing a scan, within or across a partition, we need to keep the
 +# tombstones seen in memory so we can return them to the coordinator, which
 +# will use them to make sure other replicas also know about the deleted rows.
 +# With workloads that generate a lot of tombstones, this can cause performance
 +# problems and even exaust the server heap.
 +# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets)
 +# Adjust the thresholds here if you understand the dangers and want to
 +# scan more tombstones anyway.  These thresholds may also be adjusted at runtime
 +# using the StorageService mbean.
 +tombstone_warn_threshold: 1000
 +tombstone_failure_threshold: 100000
 +
 +# Log WARN on any batch size exceeding this value. 5kb per batch by default.
 +# Caution should be taken on increasing the size of this threshold as it can lead to node
instability.
 +batch_size_warn_threshold_in_kb: 5
 +
 +# Fail any batch exceeding this value. 50kb (10x warn threshold) by default.
 +batch_size_fail_threshold_in_kb: 50
 +
 +# Log WARN on any batches not of type LOGGED than span across more partitions than this
limit
 +unlogged_batch_across_partitions_warn_threshold: 10
 +
 +# Log a warning when compacting partitions larger than this value
 +compaction_large_partition_warning_threshold_mb: 100
 +
 +# GC Pauses greater than gc_warn_threshold_in_ms will be logged at WARN level
 +# Adjust the threshold based on your application throughput requirement
 +# By default, Cassandra logs GC Pauses greater than 200 ms at INFO level
 +gc_warn_threshold_in_ms: 1000
++
+ # Maximum size of any value in SSTables. Safety measure to detect SSTable corruption
+ # early. Any value size larger than this threshold will result into marking an SSTable
+ # as corrupted.
+ # max_value_size_in_mb: 256

http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/src/java/org/apache/cassandra/config/Config.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/src/java/org/apache/cassandra/db/ClusteringPrefix.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/src/java/org/apache/cassandra/db/marshal/AbstractType.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/68d20edb/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java
index b442df0,21ce450..bd52f2f
--- a/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java
@@@ -33,7 -36,7 +36,8 @@@ import static org.junit.Assert.assertNo
  
  import org.apache.cassandra.SchemaLoader;
  import org.apache.cassandra.Util;
 +import org.apache.cassandra.cache.ChunkCache;
+ import org.apache.cassandra.config.*;
  import org.apache.cassandra.db.*;
  import org.apache.cassandra.exceptions.ConfigurationException;
  import org.apache.cassandra.io.sstable.format.SSTableReader;
@@@ -147,10 -169,8 +170,10 @@@ public class BlacklistingCompactionsTes
                  // We want to write something large enough that the corruption cannot get
undetected
                  // (even without compression)
                  byte[] corruption = new byte[corruptionSize];
-                 Arrays.fill(corruption, (byte)0xFF);
+                 random.nextBytes(corruption);
                  raf.write(corruption);
 +                if (ChunkCache.instance != null)
 +                    ChunkCache.instance.invalidateFile(sstable.getFilename());
  
              }
              finally


Mime
View raw message