cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject svn commit: r1176434 - in /cassandra/trunk: ./ bin/ conf/ contrib/ debian/ interface/thrift/gen-java/org/apache/cassandra/thrift/ redhat/ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/db/compaction/ src/java/org/apache/cassandra/secur...
Date Tue, 27 Sep 2011 15:30:59 GMT
Author: jbellis
Date: Tue Sep 27 15:30:58 2011
New Revision: 1176434

URL: http://svn.apache.org/viewvc?rev=1176434&view=rev
Log:
merge from 1.0

Modified:
    cassandra/trunk/   (props changed)
    cassandra/trunk/CHANGES.txt
    cassandra/trunk/NEWS.txt
    cassandra/trunk/bin/cassandra
    cassandra/trunk/conf/cassandra.yaml
    cassandra/trunk/contrib/   (props changed)
    cassandra/trunk/debian/cassandra.conf
    cassandra/trunk/debian/init
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java 
 (props changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java   (props
changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
  (props changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
  (props changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
  (props changed)
    cassandra/trunk/redhat/cassandra.conf
    cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
    cassandra/trunk/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
    cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionController.java
    cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
    cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
    cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
    cassandra/trunk/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
    cassandra/trunk/src/java/org/apache/cassandra/security/SSLFactory.java

Propchange: cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 27 15:30:58 2011
@@ -1,11 +1,11 @@
 /cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7:1026516-1170333,1172024
 /cassandra/branches/cassandra-0.7.0:1053690-1055654
-/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1174469,1174701,1175880
+/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1176205
 /cassandra/branches/cassandra-0.8.0:1125021-1130369
 /cassandra/branches/cassandra-0.8.1:1101014-1125018
-/cassandra/branches/cassandra-1.0:1167085-1174478,1174706,1175727,1175772,1175883,1176375
-/cassandra/branches/cassandra-1.0.0:1167104-1167229,1167232-1174472,1174704,1175725,1175770-1175771,1175882,1176371
+/cassandra/branches/cassandra-1.0:1167085-1176433
+/cassandra/branches/cassandra-1.0.0:1167104-1167229,1167232-1176432
 /cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689
 /cassandra/tags/cassandra-0.8.0-rc1:1102511-1125020
 /incubator/cassandra/branches/cassandra-0.3:774578-796573

Modified: cassandra/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1176434&r1=1176433&r2=1176434&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Tue Sep 27 15:30:58 2011
@@ -5,6 +5,8 @@
 1.0.0-final
  * Log a miningfull warning when a node receive a message for a repair session
    that don't exist anymore (CASSANDRA-3256)
+ * test for NUMA policy support as well as numactl presence (CASSANDRA-3245)
+ * Fix FD leak when internode encryption is enabled (CASSANDRA-3257)
  * Remove incorrect assertion in mergeIterator (CASSANDRA-3260)
 
 
@@ -27,8 +29,9 @@
  * Allow using quotes in "USE <keyspace>;" CLI command (CASSANDRA-3208)
  * Don't allow any cache loading exceptions to halt startup (CASSANDRA-3218)
  * Fix sstableloader --ignores option (CASSANDRA-3247)
+ * File descriptor limit increased in packaging (CASSANDRA-3206)
  * Fix deadlock in commit log during flush (CASSANDRA-3253)
-
+ 
 
 1.0.0-beta1
  * removed binarymemtable (CASSANDRA-2692)

Modified: cassandra/trunk/NEWS.txt
URL: http://svn.apache.org/viewvc/cassandra/trunk/NEWS.txt?rev=1176434&r1=1176433&r2=1176434&view=diff
==============================================================================
--- cassandra/trunk/NEWS.txt (original)
+++ cassandra/trunk/NEWS.txt Tue Sep 27 15:30:58 2011
@@ -204,7 +204,7 @@ Features
     - optional intranode encryption; see comments around 'encryption_options'
       in cassandra.yaml
     - compaction multithreading and rate-limiting; see 
-      'compaction_multithreading' and 'compaction_throughput_mb_per_sec' in
+      'concurrent_compactors' and 'compaction_throughput_mb_per_sec' in
       cassandra.yaml
     - cassandra will limit total memtable memory usage to 1/3 of the heap
       by default.  This can be ajusted or disabled with the 

Modified: cassandra/trunk/bin/cassandra
URL: http://svn.apache.org/viewvc/cassandra/trunk/bin/cassandra?rev=1176434&r1=1176433&r2=1176434&view=diff
==============================================================================
--- cassandra/trunk/bin/cassandra (original)
+++ cassandra/trunk/bin/cassandra Tue Sep 27 15:30:58 2011
@@ -91,10 +91,12 @@ fi
 
 # If numactl is available, use it. For Cassandra, the priority is to
 # avoid disk I/O. Even for the purpose of CPU efficiency, we don't
-# really have CPU<->data affinity anyway.
-if which numactl >/dev/null 2>/dev/null
+# really have CPU<->data affinity anyway. Also, empirically test that numactl
+# works before trying to use it (CASSANDRA-3245).
+NUMACTL_ARGS="--interleave=all"
+if which numactl >/dev/null 2>/dev/null && numactl $NUMACTL_ARGS ls / >/dev/null
2>/dev/null
 then
-    NUMACTL="numactl --interleave=all"
+    NUMACTL="numactl $NUMACTL_ARGS"
 else
     NUMACTL=""
 fi

Modified: cassandra/trunk/conf/cassandra.yaml
URL: http://svn.apache.org/viewvc/cassandra/trunk/conf/cassandra.yaml?rev=1176434&r1=1176433&r2=1176434&view=diff
==============================================================================
--- cassandra/trunk/conf/cassandra.yaml (original)
+++ cassandra/trunk/conf/cassandra.yaml Tue Sep 27 15:30:58 2011
@@ -137,7 +137,6 @@ concurrent_writes: 32
 # Total memory to use for memtables.  Cassandra will flush the largest
 # memtable when this much memory is used.
 # If omitted, Cassandra will set it to 1/3 of the heap.
-# If set to 0, only the old flush thresholds are used.
 # memtable_total_space_in_mb: 2048
 
 # Total space to use for commitlogs. 

Propchange: cassandra/trunk/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 27 15:30:58 2011
@@ -1,11 +1,11 @@
 /cassandra/branches/cassandra-0.6/contrib:922689-1052356,1052358-1053452,1053454,1053456-1068009
 /cassandra/branches/cassandra-0.7/contrib:1026516-1170333,1172024
 /cassandra/branches/cassandra-0.7.0/contrib:1053690-1055654
-/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1174469,1174701,1175880
+/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1175057,1175880
 /cassandra/branches/cassandra-0.8.0/contrib:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/contrib:1101014-1125018
-/cassandra/branches/cassandra-1.0/contrib:1167085-1174478,1174706,1175727,1175772,1175883,1176375
-/cassandra/branches/cassandra-1.0.0/contrib:1167104-1167229,1167232-1174472,1174704,1175725,1175770-1175771,1175882,1176371
+/cassandra/branches/cassandra-1.0/contrib:1167085-1176433
+/cassandra/branches/cassandra-1.0.0/contrib:1167104-1167229,1167232-1175725,1175770-1175771,1175882,1176371
 /cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689
 /cassandra/tags/cassandra-0.8.0-rc1/contrib:1102511-1125020
 /incubator/cassandra/branches/cassandra-0.3/contrib:774578-796573

Modified: cassandra/trunk/debian/cassandra.conf
URL: http://svn.apache.org/viewvc/cassandra/trunk/debian/cassandra.conf?rev=1176434&r1=1176433&r2=1176434&view=diff
==============================================================================
--- cassandra/trunk/debian/cassandra.conf (original)
+++ cassandra/trunk/debian/cassandra.conf Tue Sep 27 15:30:58 2011
@@ -1,2 +1,3 @@
 # Provided by the cassandra package
 cassandra  -  memlock  unlimited
+cassandra  -  nofile   100000

Modified: cassandra/trunk/debian/init
URL: http://svn.apache.org/viewvc/cassandra/trunk/debian/init?rev=1176434&r1=1176433&r2=1176434&view=diff
==============================================================================
--- cassandra/trunk/debian/init (original)
+++ cassandra/trunk/debian/init Tue Sep 27 15:30:58 2011
@@ -22,6 +22,7 @@ CONFDIR=/etc/cassandra
 JSVC=/usr/bin/jsvc
 WAIT_FOR_START=10
 CASSANDRA_HOME=/usr/share/cassandra
+FD_LIMIT=100000
 
 # The first existing directory is used for JAVA_HOME if needed.
 JVM_SEARCH_DIRS="/usr/lib/jvm/java-6-openjdk /usr/lib/jvm/java-6-sun"
@@ -127,6 +128,7 @@ do_start()
     is_running && return 1
 
     ulimit -l unlimited
+    ulimit -n "$FD_LIMIT"
 
     cassandra_home=`getent passwd cassandra | awk -F ':' '{ print $6; }'`
     cd /    # jsvc doesn't chdir() for us

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 27 15:30:58 2011
@@ -1,11 +1,11 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1170333,1172024
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1174469,1174701,1175880
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1175057,1175880
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167085-1174478,1174706,1175727,1175772,1175883,1176375
-/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167104-1167229,1167232-1174472,1174704,1175725,1175770-1175771,1175882,1176371
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167085-1176433
+/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167104-1167229,1167232-1175725,1175770-1175771,1175882,1176371
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689
 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1102511-1125020
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 27 15:30:58 2011
@@ -1,11 +1,11 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1170333,1172024
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1174469,1174701,1175880
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1175057,1175880
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167085-1174478,1174706,1175727,1175772,1175883,1176375
-/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167104-1167229,1167232-1174472,1174704,1175725,1175770-1175771,1175882,1176371
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167085-1176433
+/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167104-1167229,1167232-1175725,1175770-1175771,1175882,1176371
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689
 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1102511-1125020
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 27 15:30:58 2011
@@ -1,11 +1,11 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1170333,1172024
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1174469,1174701,1175880
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1175057,1175880
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167085-1174478,1174706,1175727,1175772,1175883,1176375
-/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167104-1167229,1167232-1174472,1174704,1175725,1175770-1175771,1175882,1176371
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167085-1176433
+/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167104-1167229,1167232-1175725,1175770-1175771,1175882,1176371
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689
 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1102511-1125020
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-796573

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 27 15:30:58 2011
@@ -1,11 +1,11 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1170333,1172024
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1174469,1174701,1175880
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1175057,1175880
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167085-1174478,1174706,1175727,1175772,1175883,1176375
-/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167104-1167229,1167232-1174472,1174704,1175725,1175770-1175771,1175882,1176371
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167085-1176433
+/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167104-1167229,1167232-1175725,1175770-1175771,1175882,1176371
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689
 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1102511-1125020
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-796573

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 27 15:30:58 2011
@@ -1,11 +1,11 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1170333,1172024
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1174469,1174701,1175880
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1175057,1175880
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167085-1174478,1174706,1175727,1175772,1175883,1176375
-/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167104-1167229,1167232-1174472,1174704,1175725,1175770-1175771,1175882,1176371
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167085-1176433
+/cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167104-1167229,1167232-1175725,1175770-1175771,1175882,1176371
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689
 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1102511-1125020
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-792198

Modified: cassandra/trunk/redhat/cassandra.conf
URL: http://svn.apache.org/viewvc/cassandra/trunk/redhat/cassandra.conf?rev=1176434&r1=1176433&r2=1176434&view=diff
==============================================================================
--- cassandra/trunk/redhat/cassandra.conf (original)
+++ cassandra/trunk/redhat/cassandra.conf Tue Sep 27 15:30:58 2011
@@ -1 +1,2 @@
 cassandra   -   memlock     unlimited
+cassandra   -   nofile      100000

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1176434&r1=1176433&r2=1176434&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Tue Sep 27 15:30:58
2011
@@ -321,7 +321,8 @@ public class ColumnFamilyStore implement
      */
     public static void scrubDataDirectories(String table, String columnFamily)
     {
-        logger.info("Removing compacted SSTable files from " + columnFamily + " (see http://wiki.apache.org/cassandra/MemtableSSTable)");
+        logger.debug("Removing compacted SSTable files from {} (see http://wiki.apache.org/cassandra/MemtableSSTable)",
columnFamily);
+
         for (Map.Entry<Descriptor,Set<Component>> sstableFiles : files(table,
columnFamily, true, true).entrySet())
         {
             Descriptor desc = sstableFiles.getKey();
@@ -884,6 +885,11 @@ public class ColumnFamilyStore implement
         return false;
     }
 
+    public boolean isKeyExistenceExpensive(Set<? extends SSTable> sstablesToIgnore)
+    {
+        return compactionStrategy.isKeyExistenceExpensive(sstablesToIgnore);
+    }
+
     /*
      * Called after a BinaryMemtable flushes its in-memory data, or we add a file
      * via bootstrap. This information is cached in the ColumnFamilyStore.

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java?rev=1176434&r1=1176433&r2=1176434&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
Tue Sep 27 15:30:58 2011
@@ -21,9 +21,11 @@ package org.apache.cassandra.db.compacti
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.SSTableReader;
 import org.apache.cassandra.service.StorageService;
 
@@ -97,4 +99,10 @@ public abstract class AbstractCompaction
      * @return size in bytes of the largest sstables for this strategy
      */
     public abstract long getMaxSSTableSize();
+
+    /**
+     * @return true if checking for whether a key exists, ignoring @param sstablesToIgnore,
+     * is going to be expensive
+     */
+    public abstract boolean isKeyExistenceExpensive(Set<? extends SSTable> sstablesToIgnore);
 }

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionController.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionController.java?rev=1176434&r1=1176433&r2=1176434&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionController.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionController.java
Tue Sep 27 15:30:58 2011
@@ -44,6 +44,7 @@ public class CompactionController
     private final boolean forceDeserialize;
 
     public final int gcBefore;
+    public boolean keyExistenceIsExpensive;
 
     public CompactionController(ColumnFamilyStore cfs, Collection<SSTableReader> sstables,
int gcBefore, boolean forceDeserialize)
     {
@@ -52,6 +53,7 @@ public class CompactionController
         this.sstables = new HashSet<SSTableReader>(sstables);
         this.gcBefore = gcBefore;
         this.forceDeserialize = forceDeserialize;
+        keyExistenceIsExpensive = cfs.getCompactionStrategy().isKeyExistenceExpensive(this.sstables);
     }
 
     public String getKeyspace()
@@ -102,13 +104,19 @@ public class CompactionController
      */
     public AbstractCompactedRow getCompactedRow(List<SSTableIdentityIterator> rows)
     {
-        if (rows.size() == 1 && !needDeserialize() && !shouldPurge(rows.get(0).getKey()))
-            return new EchoedRow(this, rows.get(0));
-
         long rowSize = 0;
         for (SSTableIdentityIterator row : rows)
             rowSize += row.dataSize;
 
+        // in-memory echoedrow is only enabled if we think checking for the key's existence
in the other sstables,
+        // is going to be less expensive than simply de/serializing the row again
+        if (rows.size() == 1 && !needDeserialize()
+            && (rowSize > DatabaseDescriptor.getInMemoryCompactionLimit() || !keyExistenceIsExpensive)
+            && !shouldPurge(rows.get(0).getKey()))
+        {
+            return new EchoedRow(this, rows.get(0));
+        }
+
         if (rowSize > DatabaseDescriptor.getInMemoryCompactionLimit())
         {
             String keyString = cfs.metadata.getKeyValidator().getString(rows.get(0).getKey().key);

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionTask.java?rev=1176434&r1=1176433&r2=1176434&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionTask.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/compaction/CompactionTask.java Tue Sep
27 15:30:58 2011
@@ -72,11 +72,12 @@ public class CompactionTask extends Abst
         Set<SSTableReader> toCompact = new HashSet<SSTableReader>(sstables);
         if (!isUserDefined)
         {
-            if ( !allowSingletonCompaction() && toCompact.size() < 2)
+            if (!allowSingletonCompaction() && toCompact.size() < 2)
             {
-                logger.info("Nothing to compact in " + cfs.getColumnFamilyName() + "." +
-                            "Use forceUserDefinedCompaction if you wish to force compaction
of single sstables " +
-                            "(e.g. for tombstone collection)");
+                String msg = "Nothing to compact in " + cfs.getColumnFamilyName();
+                if (cfs.getCompactionStrategy() instanceof SizeTieredCompactionStrategy)
+                    msg += ".  Use forceUserDefinedCompaction if you wish to force compaction
of single sstables (e.g. for tombstone collection)";
+                logger.info(msg);
                 return 0;
             }
 
@@ -220,7 +221,7 @@ public class CompactionTask extends Abst
         double mbps = dTime > 0 ? (double)endsize/(1024*1024)/((double)dTime/1000) : 0;
         logger.info(String.format("Compacted to %s.  %,d to %,d (~%d%% of original) bytes
for %,d keys at %fMBPS.  Time: %,dms.",
                                   builder.toString(), startsize, endsize, (int) (ratio *
100), totalkeysWritten, mbps, dTime));
-        logger.info(String.format("CF Total Bytes Compacted: %,d", CompactionTask.addToTotalBytesCompacted(endsize)));
+        logger.debug(String.format("CF Total Bytes Compacted: %,d", CompactionTask.addToTotalBytesCompacted(endsize)));
         return toCompact.size();
     }
 

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java?rev=1176434&r1=1176433&r2=1176434&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
Tue Sep 27 15:30:58 2011
@@ -21,17 +21,17 @@ package org.apache.cassandra.db.compacti
  */
 
 
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.SSTableReader;
 import org.apache.cassandra.notifications.INotification;
 import org.apache.cassandra.notifications.INotificationConsumer;
@@ -71,7 +71,7 @@ public class LeveledCompactionStrategy e
         maxSSTableSizeInMB = configuredMaxSSTableSize;
 
         cfs.getDataTracker().subscribe(this);
-        logger.info(this + " subscribed to the data tracker.");
+        logger.debug("{} subscribed to the data tracker.", this);
 
         manifest = LeveledManifest.create(cfs, this.maxSSTableSizeInMB);
         logger.debug("Created {}", manifest);
@@ -159,6 +159,12 @@ public class LeveledCompactionStrategy e
         return maxSSTableSizeInMB * 1024 * 1024;
     }
 
+    public boolean isKeyExistenceExpensive(Set<? extends SSTable> sstablesToIgnore)
+    {
+        Set<SSTableReader> L0 = ImmutableSet.copyOf(manifest.getLevel(0));
+        return Sets.difference(L0, sstablesToIgnore).size() + manifest.getLevelCount() >
20;
+    }
+
     @Override
     public String toString()
     {

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java?rev=1176434&r1=1176433&r2=1176434&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java Tue Sep
27 15:30:58 2011
@@ -27,8 +27,6 @@ import java.io.IOException;
 import java.util.*;
 
 import com.google.common.collect.Iterables;
-import org.apache.commons.lang.StringUtils;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -54,12 +52,13 @@ public class LeveledManifest
      * uses a pessimistic estimate of how many keys overlap (none), so we risk wasting memory
      * or even OOMing when compacting highly overlapping sstables
      */
-    private static int MAX_COMPACTING_L0 = 32;
+    static int MAX_COMPACTING_L0 = 32;
 
     private final ColumnFamilyStore cfs;
     private final List<SSTableReader>[] generations;
     private final DecoratedKey[] lastCompactedKeys;
     private final int maxSSTableSizeInMB;
+    private int levelCount;
 
     private LeveledManifest(ColumnFamilyStore cfs, int maxSSTableSizeInMB)
     {
@@ -135,14 +134,16 @@ public class LeveledManifest
         serialize();
     }
 
-    // if the number of SSTables in the current compacted set exeeds the target level, find
an empty level
+    /**
+     * if the number of SSTables in the current compacted set *by itself* exeeds the target
level's
+     * (regardless of the level's current contents), find an empty level instead
+     */
     private int skipLevels(int newLevel, Iterable<SSTableReader> added)
     {
-        // skip newlevel if the resulting sstables exceed newlevel threshold
-        if (maxBytesForLevel(newLevel) < SSTableReader.getTotalBytes(added)
-            && SSTableReader.getTotalBytes(generations[(newLevel + 1)]) == 0)
+        while (maxBytesForLevel(newLevel) < SSTableReader.getTotalBytes(added)
+            && generations[(newLevel + 1)].isEmpty())
         {
-            newLevel = skipLevels(newLevel + 1, added);
+            newLevel++;
         }
         return newLevel;
     }
@@ -171,7 +172,8 @@ public class LeveledManifest
         int newLevel = minimumLevel == maximumLevel ? maximumLevel + 1 : maximumLevel;
         newLevel = skipLevels(newLevel, added);
         assert newLevel > 0;
-        logger.debug("Adding [{}] at L{}", StringUtils.join(added.iterator(), ", "), newLevel);
+        if (logger.isDebugEnabled())
+            logger.debug("Adding [{}] at L{}", toString(added), newLevel);
 
         lastCompactedKeys[minimumLevel] = SSTable.sstableOrdering.max(added).last;
         for (SSTableReader ssTableReader : added)
@@ -185,7 +187,12 @@ public class LeveledManifest
         StringBuilder builder = new StringBuilder();
         for (SSTableReader sstable : sstables)
         {
-            builder.append(sstable.toString()).append(" (L").append(levelOf(sstable)).append("),
");
+            builder.append(sstable.descriptor.cfname)
+                   .append('-')
+                   .append(sstable.descriptor.generation)
+                   .append("(L")
+                   .append(levelOf(sstable))
+                   .append("), ");
         }
         return builder.toString();
     }
@@ -383,7 +390,7 @@ public class LeveledManifest
             File manifestFile = new File(new File(dir, cfs.table.name), cfs.columnFamily
+ ".json");
             if (manifestFile.exists())
             {
-                logger.debug("Loading manifest from {}", manifestFile);
+                logger.debug("Found manifest at {}", manifestFile);
                 return manifestFile;
             }
         }
@@ -396,4 +403,19 @@ public class LeveledManifest
     {
         return "Manifest@" + hashCode();
     }
+
+    public int getLevelCount()
+    {
+        for (int i = generations.length - 1; i >= 0; i--)
+        {
+            if (generations[i].size() > 0)
+                return i;
+        }
+        return 0;
+    }
+
+    public List<SSTableReader> getLevel(int i)
+    {
+        return generations[i];
+    }
 }

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java?rev=1176434&r1=1176433&r2=1176434&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
Tue Sep 27 15:30:58 2011
@@ -25,6 +25,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataTracker;
+import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.SSTableReader;
 import org.apache.cassandra.utils.Pair;
 
@@ -181,6 +183,11 @@ public class SizeTieredCompactionStrateg
         return Long.MAX_VALUE;
     }
 
+    public boolean isKeyExistenceExpensive(Set<? extends SSTable> sstablesToIgnore)
+    {
+        return cfs.getSSTables().size() - sstablesToIgnore.size() > 20;
+    }
+
     public String toString()
     {
         return String.format("SizeTieredCompactionStrategy[%s/%s]",

Modified: cassandra/trunk/src/java/org/apache/cassandra/security/SSLFactory.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/security/SSLFactory.java?rev=1176434&r1=1176433&r2=1176434&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/security/SSLFactory.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/security/SSLFactory.java Tue Sep 27 15:30:58
2011
@@ -25,7 +25,6 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
-import java.net.SocketAddress;
 import java.security.KeyStore;
 
 import javax.net.ssl.KeyManagerFactory;
@@ -35,6 +34,7 @@ import javax.net.ssl.SSLSocket;
 import javax.net.ssl.TrustManagerFactory;
 
 import org.apache.cassandra.config.EncryptionOptions;
+import org.apache.cassandra.io.util.FileUtils;
 
 /**
  * A Factory for providing and setting up Client and Server SSL wrapped
@@ -46,7 +46,6 @@ public final class SSLFactory
     private static final String ALGORITHM = "SunX509";
     private static final String STORE_TYPE = "JKS";
 
-
     public static SSLServerSocket getServerSocket(EncryptionOptions options, InetAddress
address, int port) throws IOException
     {
         SSLContext ctx = createSSLContext(options);
@@ -75,28 +74,39 @@ public final class SSLFactory
         return socket;
     }
 
-    private static SSLContext createSSLContext(EncryptionOptions options) throws IOException
{
+    private static SSLContext createSSLContext(EncryptionOptions options) throws IOException
+    {
+        FileInputStream tsf = new FileInputStream(options.truststore);
+        FileInputStream ksf = new FileInputStream(options.keystore);
         SSLContext ctx;
-        try {
+        try
+        {
             ctx = SSLContext.getInstance(PROTOCOL);
-            TrustManagerFactory tmf = null;
-            KeyManagerFactory kmf = null;
+            TrustManagerFactory tmf;
+            KeyManagerFactory kmf;
 
             tmf = TrustManagerFactory.getInstance(ALGORITHM);
             KeyStore ts = KeyStore.getInstance(STORE_TYPE);
-            ts.load(new FileInputStream(options.truststore), options.truststore_password.toCharArray());
+            ts.load(tsf, options.truststore_password.toCharArray());
             tmf.init(ts);
 
             kmf = KeyManagerFactory.getInstance(ALGORITHM);
             KeyStore ks = KeyStore.getInstance(STORE_TYPE);
-            ks.load(new FileInputStream(options.keystore), options.keystore_password.toCharArray());
+            ks.load(ksf, options.keystore_password.toCharArray());
             kmf.init(ks, options.keystore_password.toCharArray());
 
             ctx.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
 
-        } catch (Exception e) {
+        }
+        catch (Exception e)
+        {
             throw new IOException("Error creating the initializing the SSL Context", e);
         }
+        finally
+        {
+            FileUtils.closeQuietly(tsf);
+            FileUtils.closeQuietly(ksf);
+        }
         return ctx;
     }
 }



Mime
View raw message