cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject svn commit: r1070541 - in /cassandra/trunk: ./ bin/ conf/ contrib/ debian/ interface/thrift/gen-java/org/apache/cassandra/thrift/ src/java/org/apache/cassandra/cli/ src/java/org/apache/cassandra/config/ src/java/org/apache/cassandra/db/ src/java/org/ap...
Date Mon, 14 Feb 2011 16:06:51 GMT
Author: jbellis
Date: Mon Feb 14 16:06:50 2011
New Revision: 1070541

URL: http://svn.apache.org/viewvc?rev=1070541&view=rev
Log:
merge from 0.7

Modified:
    cassandra/trunk/   (props changed)
    cassandra/trunk/CHANGES.txt
    cassandra/trunk/NEWS.txt
    cassandra/trunk/bin/sstablekeys
    cassandra/trunk/conf/cassandra.yaml
    cassandra/trunk/contrib/   (props changed)
    cassandra/trunk/debian/changelog
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java   (props changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java   (props changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java   (props changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java   (props changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java   (props changed)
    cassandra/trunk/src/java/org/apache/cassandra/cli/Cli.g
    cassandra/trunk/src/java/org/apache/cassandra/cli/CliClient.java
    cassandra/trunk/src/java/org/apache/cassandra/cli/CliMain.java
    cassandra/trunk/src/java/org/apache/cassandra/cli/CliOptions.java
    cassandra/trunk/src/java/org/apache/cassandra/cli/CliUserHelp.java
    cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java
    cassandra/trunk/src/java/org/apache/cassandra/config/Config.java
    cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
    cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
    cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java
    cassandra/trunk/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
    cassandra/trunk/src/java/org/apache/cassandra/io/sstable/CacheWriter.java
    cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
    cassandra/trunk/src/java/org/apache/cassandra/net/MessagingService.java
    cassandra/trunk/src/java/org/apache/cassandra/service/AbstractCassandraDaemon.java
    cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java
    cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java
    cassandra/trunk/src/java/org/apache/cassandra/tools/NodeCmd.java
    cassandra/trunk/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
    cassandra/trunk/src/java/org/apache/cassandra/utils/StatusLogger.java
    cassandra/trunk/test/unit/org/apache/cassandra/cli/CliTest.java

Propchange: cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 14 16:06:50 2011
@@ -1,5 +1,5 @@
-/cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1068009
-/cassandra/branches/cassandra-0.7:1026516-1068562
+/cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1068009,1068978
+/cassandra/branches/cassandra-0.7:1026516-1070530
 /cassandra/branches/cassandra-0.7.0:1053690-1055654
 /cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689
 /incubator/cassandra/branches/cassandra-0.3:774578-796573

Modified: cassandra/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Mon Feb 14 16:06:50 2011
@@ -11,8 +11,9 @@
 
 
 0.7.2
- * cache writing moved to CompactionManager to reduce i/o contention and
-   updated to use non-cache-polluting writes (CASSANDRA-2053)
+ * copy DecoratedKey.key when inserting into caches to avoid retaining
+   a reference to the underlying buffer (CASSANDRA-2102)
+ * format subcolumn names with subcomparator (CASSANDRA-2136)
 
 
 0.7.1
@@ -67,6 +68,19 @@
  * avoid blocking gossip while deleting handoff hints (CASSANDRA-2073)
  * ignore messages from newer versions, keep track of nodes in gossip 
    regardless of version (CASSANDRA-1970)
+ * cache writing moved to CompactionManager to reduce i/o contention and
+   updated to use non-cache-polluting writes (CASSANDRA-2053)
+ * page through large rows when exporting to JSON (CASSANDRA-2041)
+ * add flush_largest_memtables_at and reduce_cache_sizes_at options
+   (CASSANDRA-2142)
+ * add cli 'describe cluster' command (CASSANDRA-2127)
+ * add cli support for setting username/password at 'connect' command 
+   (CASSANDRA-2111)
+ * add -D option to Stress.java to allow reading hosts from a file 
+   (CASSANDRA-2149)
+ * bound hints CF throughput between 32M and 256M (CASSANDRA-2148)
+ * continue starting when invalid saved cache entries are encountered
+   (CASSANDRA-2076)
 
 
 0.7.0-final
@@ -214,6 +228,8 @@
  * fix service initialization order deadlock (CASSANDRA-1756)
  * multi-line cli commands (CASSANDRA-1742)
  * fix race between snapshot and compaction (CASSANDRA-1736)
+ * add listEndpointsPendingHints, deleteHintsForEndpoint JMX methods 
+   (CASSANDRA-1551)
 
 
 0.7.0-beta3

Modified: cassandra/trunk/NEWS.txt
URL: http://svn.apache.org/viewvc/cassandra/trunk/NEWS.txt?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/NEWS.txt (original)
+++ cassandra/trunk/NEWS.txt Mon Feb 14 16:06:50 2011
@@ -27,17 +27,33 @@ JMX
 0.7.1
 =====
 
-Uprading
---------
+Upgrading
+---------
     - 0.7.1 is completely backwards compatible with 0.7.0.  Just restart
       each node with the new version, one at a time.  (The cluster does
       not all need to be upgraded simultaneously.)
 
 Features
 --------
-    - Cassandra can perform writes efficiently across datacenters by
+    - added flush_largest_memtables_at and reduce_cache_sizes_at options
+      to cassandra.yaml as an escape valve for memory pressure
+    - added option to specify -Dcassandra.join_ring=false on startup
+      to allow "warm spare" nodes or performing JMX maintenance before
+      joining the ring
+
+Performance
+-----------
+    - Disk writes and sequential scans avoid polluting page cache
+      (requires JNA to be enabled)
+    - Cassandra performs writes efficiently across datacenters by
       sending a single copy of the mutation and having the recipient
       forward that to other replicas in its datacenter.
+    - Improved network buffering
+    - Reduced lock contention on memtable flush
+    - Optimized supercolumn deserialization
+    - Zero-copy reads from mmapped sstable files
+    - Explicitly set higher JVM new generation size
+    - Reduced i/o contention during saving of caches
 
 
 0.7.0

Modified: cassandra/trunk/bin/sstablekeys
URL: http://svn.apache.org/viewvc/cassandra/trunk/bin/sstablekeys?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/bin/sstablekeys (original)
+++ cassandra/trunk/bin/sstablekeys Mon Feb 14 16:06:50 2011
@@ -50,6 +50,6 @@ fi
 
 $JAVA -cp $CLASSPATH  -Dstorage-config=$CASSANDRA_CONF \
         -Dlog4j.configuration=log4j-tools.properties \
-        org.apache.cassandra.tools.SSTableExport "$1" -e
+        org.apache.cassandra.tools.SSTableExport "$@" -e
 
 # vi:ai sw=4 ts=4 tw=0 et

Modified: cassandra/trunk/conf/cassandra.yaml
URL: http://svn.apache.org/viewvc/cassandra/trunk/conf/cassandra.yaml?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/conf/cassandra.yaml (original)
+++ cassandra/trunk/conf/cassandra.yaml Mon Feb 14 16:06:50 2011
@@ -102,6 +102,31 @@ seed_provider:
           # seeds is actually a comma-delimited list of addresses.
           - seeds: "127.0.0.1"
 
+# emergency pressure valve: each time heap usage after a full (CMS)
+# garbage collection is above this fraction of the max, Cassandra will
+# flush the largest memtables.  
+#
+# Set to 1.0 to disable.  Setting this lower than
+# CMSInitiatingOccupancyFraction is not likely to be useful.
+#
+# RELYING ON THIS AS YOUR PRIMARY TUNING MECHANISM WILL WORK POORLY:
+# it is most effective under light to moderate load, or read-heavy
+# workloads; under truly massive write load, it will often be too
+# little, too late.
+flush_largest_memtables_at: 0.75
+
+# emergency pressure valve #2: the first time heap usage after a full
+# (CMS) garbage collection is above this fraction of the max,
+# Cassandra will reduce cache maximum _capacity_ to the given fraction
+# of the current _size_.  Should usually be set substantially above
+# flush_largest_memtables_at, since that will have less long-term
+# impact on the system.  
+# 
+# Set to 1.0 to disable.  Setting this lower than
+# CMSInitiatingOccupancyFraction is not likely to be useful.
+reduce_cache_sizes_at: 0.85
+reduce_cache_capacity_to: 0.6
+
 # Access mode.  mmapped i/o is substantially faster, but only practical on
 # a 64bit machine (which notably does not include EC2 "small" instances)
 # or relatively small datasets.  "auto", the safe choice, will enable

Propchange: cassandra/trunk/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 14 16:06:50 2011
@@ -1,5 +1,5 @@
 /cassandra/branches/cassandra-0.6/contrib:922689-1052356,1052358-1053452,1053454,1053456-1068009
-/cassandra/branches/cassandra-0.7/contrib:1026516-1070510
+/cassandra/branches/cassandra-0.7/contrib:1026516-1070530
 /cassandra/branches/cassandra-0.7.0/contrib:1053690-1055654
 /cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689
 /incubator/cassandra/branches/cassandra-0.3/contrib:774578-796573

Modified: cassandra/trunk/debian/changelog
URL: http://svn.apache.org/viewvc/cassandra/trunk/debian/changelog?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/debian/changelog (original)
+++ cassandra/trunk/debian/changelog Mon Feb 14 16:06:50 2011
@@ -2,7 +2,7 @@ cassandra (0.7.1) unstable; urgency=low
 
   * New stable point release.
 
- -- Eric Evans <eevans@apache.org>  Fri, 04 Feb 2011 12:57:52 -0600
+ -- Eric Evans <eevans@apache.org>  Thu, 10 Feb 2011 10:34:50 -0600
 
 cassandra (0.7.0~rc4) unstable; urgency=low
 

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 14 16:06:50 2011
@@ -1,5 +1,5 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1068009
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1068562
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1068009,1068978
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1070530
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 14 16:06:50 2011
@@ -1,5 +1,5 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1068009
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1068562
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1068009,1068978
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1070530
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 14 16:06:50 2011
@@ -1,5 +1,5 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1068009
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1068562
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1068009,1068978
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1070530
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-796573

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 14 16:06:50 2011
@@ -1,5 +1,5 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1068009
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1068562
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1068009,1068978
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1070530
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-796573

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb 14 16:06:50 2011
@@ -1,5 +1,5 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1068009
-/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1068562
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1068009,1068978
+/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1070530
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-792198

Modified: cassandra/trunk/src/java/org/apache/cassandra/cli/Cli.g
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/cli/Cli.g?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/cli/Cli.g (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/cli/Cli.g Mon Feb 14 16:06:50 2011
@@ -36,6 +36,7 @@ tokens {
     //
     NODE_CONNECT;
     NODE_DESCRIBE_TABLE;
+    NODE_DESCRIBE_CLUSTER;
     NODE_USE_TABLE;
     NODE_EXIT;
     NODE_HELP;
@@ -138,6 +139,7 @@ statement
     | exitStatement
     | countStatement
     | describeTable
+    | describeCluster
     | addKeyspace
     | addColumnFamily
     | updateKeyspace
@@ -157,10 +159,10 @@ statement
     ;
 
 connectStatement
-    : CONNECT host '/' port 
-        -> ^(NODE_CONNECT host port)
-    | CONNECT ip_address '/' port 
-        -> ^(NODE_CONNECT ip_address port)
+    : CONNECT host '/' port (username password)?
+        -> ^(NODE_CONNECT host port (username password)?)
+    | CONNECT ip_address '/' port (username password)?
+        -> ^(NODE_CONNECT ip_address port (username password)?)
     ;
 
 helpStatement
@@ -172,6 +174,8 @@ helpStatement
         -> ^(NODE_HELP NODE_USE_TABLE)
     | HELP DESCRIBE KEYSPACE 
         -> ^(NODE_HELP NODE_DESCRIBE_TABLE)
+    | HELP DESCRIBE 'CLUSTER'
+        -> ^(NODE_HELP NODE_DESCRIBE_CLUSTER)
     | HELP EXIT 
         -> ^(NODE_HELP NODE_EXIT)
     | HELP QUIT 
@@ -325,6 +329,11 @@ describeTable
         -> ^(NODE_DESCRIBE_TABLE (keyspace)?)
     ;
     
+describeCluster
+    : DESCRIBE 'CLUSTER'
+        -> ^(NODE_DESCRIBE_CLUSTER)
+    ;
+
 useKeyspace
     : USE keyspace ( username )? ( password )? 
         -> ^(NODE_USE_TABLE keyspace ( username )? ( password )?)

Modified: cassandra/trunk/src/java/org/apache/cassandra/cli/CliClient.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/cli/CliClient.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/cli/CliClient.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/cli/CliClient.java Mon Feb 14 16:06:50 2011
@@ -173,6 +173,9 @@ public class CliClient extends CliUserHe
                 case CliParser.NODE_DESCRIBE_TABLE:
                     executeDescribeKeySpace(tree);
                     break;
+                case CliParser.NODE_DESCRIBE_CLUSTER:
+                    executeDescribeCluster();
+                    break;
                 case CliParser.NODE_USE_TABLE:
                     executeUseKeySpace(tree);
                     break;
@@ -314,7 +317,8 @@ public class CliClient extends CliUserHe
 
         AbstractType validator;
         CfDef cfDef = getCfDef(columnFamily);
-        
+        boolean isSuperCF = cfDef.column_type.equals("Super");
+
         // Print out super columns or columns.
         for (ColumnOrSuperColumn cosc : columns)
         {
@@ -337,9 +341,16 @@ public class CliClient extends CliUserHe
             {
                 Column column = cosc.column;
                 validator = getValidatorForValue(cfDef, column.getName());
-                sessionState.out.printf("=> (column=%s, value=%s, timestamp=%d%s)%n", formatColumnName(keyspace, columnFamily, column),
-                                                validator.getString(column.value), column.timestamp,
-                                                column.isSetTtl() ? String.format(", ttl=%d", column.getTtl()) : "");
+
+                String formattedName = isSuperCF
+                                       ? formatSubcolumnName(keyspace, columnFamily, column)
+                                       : formatColumnName(keyspace, columnFamily, column);
+
+                sessionState.out.printf("=> (column=%s, value=%s, timestamp=%d%s)%n",
+                                        formattedName,
+                                        validator.getString(column.value),
+                                        column.timestamp,
+                                        column.isSetTtl() ? String.format(", ttl=%d", column.getTtl()) : "");
             }
         }
         
@@ -461,9 +472,15 @@ public class CliClient extends CliUserHe
             valueAsString = (validator == null) ? new String(columnValue, Charsets.UTF_8) : validator.getString(ByteBuffer.wrap(columnValue));
         }
 
+        String formattedColumnName = isSuper
+                                     ? formatSubcolumnName(keySpace, columnFamily, column)
+                                     : formatColumnName(keySpace, columnFamily, column);
+
         // print results
         sessionState.out.printf("=> (column=%s, value=%s, timestamp=%d%s)%n",
-                                formatColumnName(keySpace, columnFamily, column), valueAsString, column.timestamp,
+                                formattedColumnName,
+                                valueAsString,
+                                column.timestamp,
                                 column.isSetTtl() ? String.format(", ttl=%d", column.getTtl()) : "");
     }
 
@@ -880,7 +897,12 @@ public class CliClient extends CliUserHe
                 cfDef.setKey_cache_size(Double.parseDouble(mValue));
                 break;
             case READ_REPAIR_CHANCE:
-                cfDef.setRead_repair_chance(Double.parseDouble(mValue));
+                double chance = Double.parseDouble(mValue) / 100;
+
+                if (chance > 1)
+                    throw new RuntimeException("Error: read_repair_chance / 100 should not be greater than 1.");
+
+                cfDef.setRead_repair_chance(chance);
                 break;
             case GC_GRACE:
                 cfDef.setGc_grace_seconds(Integer.parseInt(mValue));
@@ -1412,12 +1434,39 @@ public class CliClient extends CliUserHe
         describeKeySpace(keySpaceName, null);
     }
 
+    // ^(NODE_DESCRIBE_CLUSTER) or describe: schema_versions, partitioner, snitch
+    private void executeDescribeCluster()
+    {
+        if (!CliMain.isConnected())
+            return;
+
+        sessionState.out.println("Cluster Information:");
+        try
+        {
+            sessionState.out.println("   Snitch: " + thriftClient.describe_snitch());
+            sessionState.out.println("   Partitioner: " + thriftClient.describe_partitioner());
+
+            sessionState.out.println("   Schema versions: ");
+            Map<String,List<String>> versions = thriftClient.describe_schema_versions();
+
+            for (String version : versions.keySet())
+            {
+                sessionState.out.println("\t" + version + ": " + versions.get(version));
+            }
+        }
+        catch (Exception e)
+        {
+            String message = (e instanceof InvalidRequestException) ? ((InvalidRequestException) e).getWhy() : e.getMessage();
+            sessionState.err.println("Error retrieving data: " + message);
+        }
+    }
+
     // process a statement of the form: connect hostname/port
     private void executeConnect(Tree statement)
     {
         Tree idList = statement.getChild(0);
         int portNumber = Integer.parseInt(statement.getChild(1).getText());
-        
+
         StringBuilder hostName = new StringBuilder();
         int idCount = idList.getChildCount(); 
         for (int idx = 0; idx < idCount; idx++)
@@ -1432,6 +1481,14 @@ public class CliClient extends CliUserHe
         // now, connect to the newly specified host name and port
         sessionState.hostName = hostName.toString();
         sessionState.thriftPort = portNumber;
+
+        // if we have user name and password
+        if (statement.getChildCount() == 4)
+        {
+            sessionState.username = statement.getChild(2).getText();
+            sessionState.password = CliUtils.unescapeSQLString(statement.getChild(3).getText());
+        }
+
         CliMain.connect(sessionState.hostName, sessionState.thriftPort);
     }
 

Modified: cassandra/trunk/src/java/org/apache/cassandra/cli/CliMain.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/cli/CliMain.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/cli/CliMain.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/cli/CliMain.java Mon Feb 14 16:06:50 2011
@@ -115,35 +115,38 @@ public class CliMain
                 return;
             }
         }
-        
+
         if ((sessionState.username != null) && (sessionState.password != null))
         {
-            // Authenticate 
+            // Authenticate
             Map<String, String> credentials = new HashMap<String, String>();
             credentials.put(SimpleAuthenticator.USERNAME_KEY, sessionState.username);
             credentials.put(SimpleAuthenticator.PASSWORD_KEY, sessionState.password);
             AuthenticationRequest authRequest = new AuthenticationRequest(credentials);
-            try 
+            try
             {
                 thriftClient.login(authRequest);
                 cliClient.setUsername(sessionState.username);
-            } 
-            catch (AuthenticationException e) 
+            }
+            catch (AuthenticationException e)
             {
+                thriftClient = null;
                 sessionState.err.println("Exception during authentication to the cassandra node, " +
                 		"Verify the keyspace exists, and that you are using the correct credentials.");
                 return;
-            } 
-            catch (AuthorizationException e) 
+            }
+            catch (AuthorizationException e)
             {
+                thriftClient = null;
                 sessionState.err.println("You are not authorized to use keyspace: " + sessionState.keyspace);
                 return;
             }
-            catch (TException e) 
+            catch (TException e)
             {
+                thriftClient = null;
                 sessionState.err.println("Login failure. Did you specify 'keyspace', 'username' and 'password'?");
                 return;
-            } 
+            }
         }
         
         // Lookup the cluster name, this is to make it clear which cluster the user is connected to
@@ -155,7 +158,6 @@ public class CliMain
         }
         catch (Exception e)
         {
-
             sessionState.err.println("Exception retrieving information about the cassandra node, check you have connected to the thrift port.");
 
             if (sessionState.debug)

Modified: cassandra/trunk/src/java/org/apache/cassandra/cli/CliOptions.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/cli/CliOptions.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/cli/CliOptions.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/cli/CliOptions.java Mon Feb 14 16:06:50 2011
@@ -124,11 +124,19 @@ public class CliOptions
             {
             	css.username = cmd.getOptionValue(USERNAME_OPTION);
             }
+            else
+            {
+                css.username = "default";
+            }
 
             if (cmd.hasOption(PASSWORD_OPTION))
             {
             	css.password = cmd.getOptionValue(PASSWORD_OPTION);
             }
+            else
+            {
+                css.password = "";
+            }
 
             // Look for keyspace
             if (cmd.hasOption(KEYSPACE_OPTION))

Modified: cassandra/trunk/src/java/org/apache/cassandra/cli/CliUserHelp.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/cli/CliUserHelp.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/cli/CliUserHelp.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/cli/CliUserHelp.java Mon Feb 14 16:06:50 2011
@@ -83,10 +83,12 @@ public class CliUserHelp {
                 state.out.println("Display the general help page with a list of available commands.");
                 break;
             case CliParser.NODE_CONNECT:
-                state.out.println("connect <hostname>/<port>;\n");
-                state.out.println("Connect to the specified host on the specified port.\n");
+                state.out.println("connect <hostname>/<port> (<username> '<password>')?;\n");
+                state.out.println("Connect to the specified host on the specified port (using specified username and password).\n");
                 state.out.println("example:");
                 state.out.println("connect localhost/9160;");
+                state.out.println("connect localhost/9160 user 'badpasswd';");
+                state.out.println("connect 127.0.0.1/9160 user 'badpasswd';");
                 break;
 
             case CliParser.NODE_USE_TABLE:
@@ -104,6 +106,11 @@ public class CliUserHelp {
                 state.out.println("describe keyspace system;");
                 break;
 
+            case CliParser.NODE_DESCRIBE_CLUSTER:
+                state.out.println("describe cluster;\n");
+                state.out.println("Display information about cluster: snitch, partitioner, schema versions.");
+                break;
+
             case CliParser.NODE_EXIT:
                 state.out.println("exit;");
                 state.out.println("quit;\n");
@@ -330,11 +337,12 @@ public class CliUserHelp {
             state.out.println("?                                                          Display this message.");
             state.out.println("help;                                                         Display this help.");
             state.out.println("help <command>;                         Display detailed, command-specific help.");
-            state.out.println("connect <hostname>/<port>;                            Connect to thrift service.");
+            state.out.println("connect <hostname>/<port> (<username> '<password>')?; Connect to thrift service.");
             state.out.println("use <keyspace> [<username> 'password'];                    Switch to a keyspace.");
             state.out.println("describe keyspace (<keyspacename>)?;                          Describe keyspace.");
             state.out.println("exit;                                                                  Exit CLI.");
             state.out.println("quit;                                                                  Exit CLI.");
+            state.out.println("describe cluster;                             Display information about cluster.");
             state.out.println("show cluster name;                                         Display cluster name.");
             state.out.println("show keyspaces;                                          Show list of keyspaces.");
             state.out.println("show api version;                                       Show server API version.");

Modified: cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java Mon Feb 14 16:06:50 2011
@@ -46,10 +46,12 @@ import org.apache.cassandra.utils.Pair;
 
 public final class CFMetaData
 {
+
     public final static double DEFAULT_ROW_CACHE_SIZE = 0.0;
     public final static double DEFAULT_KEY_CACHE_SIZE = 200000;
     public final static double DEFAULT_READ_REPAIR_CHANCE = 1.0;
     public final static boolean DEFAULT_REPLICATE_ON_WRITE = false;
+    public final static int DEFAULT_SYSTEM_MEMTABLE_THROUGHPUT_IN_MB = 8;
     public final static int DEFAULT_ROW_CACHE_SAVE_PERIOD_IN_SECONDS = 0;
     public final static int DEFAULT_KEY_CACHE_SAVE_PERIOD_IN_SECONDS = 4 * 3600;
     public final static int DEFAULT_GC_GRACE_SECONDS = 864000;
@@ -65,13 +67,13 @@ public final class CFMetaData
     
     private static final BiMap<Pair<String, String>, Integer> cfIdMap = HashBiMap.create();
     
-    public static final CFMetaData StatusCf = newSystemTable(SystemTable.STATUS_CF, 0, "persistent metadata for the local node", BytesType.instance, null);
-    public static final CFMetaData HintsCf = newSystemTable(HintedHandOffManager.HINTS_CF, 1, "hinted handoff data", BytesType.instance, BytesType.instance);
-    public static final CFMetaData MigrationsCf = newSystemTable(Migration.MIGRATIONS_CF, 2, "individual schema mutations", TimeUUIDType.instance, null);
-    public static final CFMetaData SchemaCf = newSystemTable(Migration.SCHEMA_CF, 3, "current state of the schema", UTF8Type.instance, null);
-    public static final CFMetaData IndexCf = newSystemTable(SystemTable.INDEX_CF, 5, "indexes that have been completed", UTF8Type.instance, null);
+    public static final CFMetaData StatusCf = newSystemTable(SystemTable.STATUS_CF, 0, "persistent metadata for the local node", BytesType.instance, null, DEFAULT_SYSTEM_MEMTABLE_THROUGHPUT_IN_MB);
+    public static final CFMetaData HintsCf = newSystemTable(HintedHandOffManager.HINTS_CF, 1, "hinted handoff data", BytesType.instance, BytesType.instance, Math.min(256, Math.max(32, DEFAULT_MEMTABLE_THROUGHPUT_IN_MB / 2)));
+    public static final CFMetaData MigrationsCf = newSystemTable(Migration.MIGRATIONS_CF, 2, "individual schema mutations", TimeUUIDType.instance, null, DEFAULT_SYSTEM_MEMTABLE_THROUGHPUT_IN_MB);
+    public static final CFMetaData SchemaCf = newSystemTable(Migration.SCHEMA_CF, 3, "current state of the schema", UTF8Type.instance, null, DEFAULT_SYSTEM_MEMTABLE_THROUGHPUT_IN_MB);
+    public static final CFMetaData IndexCf = newSystemTable(SystemTable.INDEX_CF, 5, "indexes that have been completed", UTF8Type.instance, null, DEFAULT_SYSTEM_MEMTABLE_THROUGHPUT_IN_MB);
 
-    private static CFMetaData newSystemTable(String cfName, int cfId, String comment, AbstractType comparator, AbstractType subComparator)
+    private static CFMetaData newSystemTable(String cfName, int cfId, String comment, AbstractType comparator, AbstractType subComparator, int memtableThroughPutInMB)
     {
         return new CFMetaData(Table.SYSTEM_TABLE,
                               cfName,
@@ -90,8 +92,8 @@ public final class CFMetaData
                               DEFAULT_ROW_CACHE_SAVE_PERIOD_IN_SECONDS,
                               DEFAULT_KEY_CACHE_SAVE_PERIOD_IN_SECONDS,
                               DEFAULT_MEMTABLE_LIFETIME_IN_MINS,
-                              DEFAULT_MEMTABLE_THROUGHPUT_IN_MB,
-                              DEFAULT_MEMTABLE_OPERATIONS_IN_MILLIONS,
+                              memtableThroughPutInMB,
+                              sizeMemtableOperations(memtableThroughPutInMB),
                               cfId,
                               Collections.<ByteBuffer, ColumnDefinition>emptyMap());
     }

Modified: cassandra/trunk/src/java/org/apache/cassandra/config/Config.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/Config.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/Config.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/Config.java Mon Feb 14 16:06:50 2011
@@ -106,7 +106,10 @@ public class Config
     public Integer index_interval = 128;
 
     public List<RawKeyspace> keyspaces;
-    
+    public Double flush_largest_memtables_at = 1.0;
+    public Double reduce_cache_sizes_at = 1.0;
+    public double reduce_cache_capacity_to = 0.6;
+
     public static enum CommitLogSync {
         periodic,
         batch

Modified: cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java Mon Feb 14 16:06:50 2011
@@ -20,7 +20,6 @@ package org.apache.cassandra.config;
 
 import java.io.*;
 import java.net.InetAddress;
-import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
@@ -1177,4 +1176,19 @@ public class DatabaseDescriptor
     {
         return conf.encryption_options;
     }
+
+    public static double getFlushLargestMemtablesAt()
+    {
+        return conf.flush_largest_memtables_at;
+    }
+
+    public static double getReduceCacheSizesAt()
+    {
+        return conf.reduce_cache_sizes_at;
+    }
+
+    public static double getReduceCacheCapacityTo()
+    {
+        return conf.reduce_cache_capacity_to;
+    }
 }

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Mon Feb 14 16:06:50 2011
@@ -258,7 +258,19 @@ public class ColumnFamilyStore implement
                     int size = in.readInt();
                     byte[] bytes = new byte[size];
                     in.readFully(bytes);
-                    keys.add(StorageService.getPartitioner().decorateKey(ByteBuffer.wrap(bytes)));
+                    ByteBuffer buffer = ByteBuffer.wrap(bytes);
+                    DecoratedKey key;
+                    try
+                    {
+                        key = StorageService.getPartitioner().decorateKey(buffer);
+                    }
+                    catch (Exception e)
+                    {
+                        logger.info(String.format("unable to read entry #%s from saved cache %s; skipping remaining entries",
+                                                  keys.size(), path.getAbsolutePath()), e);
+                        break;
+                    }
+                    keys.add(key);
                 }
                 if (logger.isDebugEnabled())
                     logger.debug(String.format("completed reading (%d ms; %d keys) saved cache %s",
@@ -465,7 +477,8 @@ public class ColumnFamilyStore implement
             generations.add(desc.generation);
             if (desc.isFromTheFuture())
             {
-                throw new RuntimeException("you can't open sstables from the future!");
+                throw new RuntimeException(String.format("Can't open sstables from the future! Current version %s, found file: %s",
+                                                         Descriptor.CURRENT_VERSION, desc));
             }
         }
         Collections.sort(generations);
@@ -1182,7 +1195,8 @@ public class ColumnFamilyStore implement
                 }
             }
 
-            ssTables.getRowCache().put(key, cached);
+            // avoid keeping a permanent reference to the original key buffer
+            ssTables.getRowCache().put(new DecoratedKey(key.token, ByteBufferUtil.clone(key.key)), cached);
         }
         return cached;
     }
@@ -1973,7 +1987,7 @@ public class ColumnFamilyStore implement
     public String toString()
     {
         return "ColumnFamilyStore(" +
-               "table='" + table + '\'' +
+               "table='" + table.name + '\'' +
                ", columnFamily='" + columnFamily + '\'' +
                ')';
     }
@@ -2114,4 +2128,26 @@ public class ColumnFamilyStore implement
     {
         return partitioner instanceof LocalPartitioner;
     }
+
+    /**
+     * sets each cache's maximum capacity to 75% of its current size
+     */
+    public void reduceCacheSizes()
+    {
+        if (ssTables.getRowCache().getCapacity() > 0)
+        {
+            int newCapacity = (int) (DatabaseDescriptor.getReduceCacheCapacityTo() * ssTables.getRowCache().getSize());
+            logger.warn(String.format("Reducing %s row cache capacity from %d to %s to reduce memory pressure",
+                                      columnFamily, ssTables.getRowCache().getCapacity(), newCapacity));
+            ssTables.getRowCache().setCapacity(newCapacity);
+        }
+
+        if (ssTables.getKeyCache().getCapacity() > 0)
+        {
+            int newCapacity = (int) (DatabaseDescriptor.getReduceCacheCapacityTo() * ssTables.getKeyCache().getSize());
+            logger.warn(String.format("Reducing %s key cache capacity from %d to %s to reduce memory pressure",
+                                      columnFamily, ssTables.getKeyCache().getCapacity(), newCapacity));
+            ssTables.getKeyCache().setCapacity(newCapacity);
+        }
+    }
 }

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java Mon Feb 14 16:06:50 2011
@@ -491,7 +491,7 @@ public class CompactionManager implement
             long expectedRangeFileSize = cfs.getExpectedCompactedFileSize(Arrays.asList(sstable)) / 2;
             String compactionFileLocation = table.getDataFileLocation(expectedRangeFileSize);
             if (compactionFileLocation == null)
-                throw new UnsupportedOperationException("disk full");
+                throw new IOException("disk full");
 
             long startTime = System.currentTimeMillis();
             long totalkeysWritten = 0;

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/commitlog/CommitLog.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/commitlog/CommitLog.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/commitlog/CommitLog.java Mon Feb 14 16:06:50 2011
@@ -211,16 +211,21 @@ public class CommitLog
                         logger.debug("Reading mutation at " + reader.getFilePointer());
 
                     long claimedCRC32;
-
                     Checksum checksum = new CRC32();
                     int serializedSize;
                     try
                     {
                         // any of the reads may hit EOF
                         serializedSize = reader.readInt();
+                        // RowMutation must be at LEAST 10 bytes:
+                        // 3 each for a non-empty Table and Key (including the 2-byte length from
+                        // writeUTF/writeWithShortLength) and 4 bytes for column count.
+                        // This prevents CRC by being fooled by special-case garbage in the file; see CASSANDRA-2128
+                        if (serializedSize < 10)
+                            break;
                         long claimedSizeChecksum = reader.readLong();
                         checksum.update(serializedSize);
-                        if (checksum.getValue() != claimedSizeChecksum || serializedSize <= 0)
+                        if (checksum.getValue() != claimedSizeChecksum)
                             break; // entry wasn't synced correctly/fully.  that's ok.
 
                         if (serializedSize > bytes.length)

Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/CacheWriter.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/CacheWriter.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/CacheWriter.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/CacheWriter.java Mon Feb 14 16:06:50 2011
@@ -1,4 +1,25 @@
 package org.apache.cassandra.io.sstable;
+/*
+ * 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * 
+ */
+
 
 import java.io.File;
 import java.io.IOException;
@@ -6,7 +27,6 @@ import java.nio.ByteBuffer;
 import java.util.Set;
 
 import com.google.common.base.Function;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -23,7 +43,7 @@ public class CacheWriter<K, V> implement
     private final Function<K, ByteBuffer> converter;
     private final Set<K> keys;
     private final String columnFamily;
-    private final long totalBytes;
+    private final long estimatedTotalBytes;
     private long bytesWritten;
 
     public CacheWriter(String columnFamily, JMXInstrumentedCache<K, V> cache, File path, Function<K, ByteBuffer> converter)
@@ -36,7 +56,9 @@ public class CacheWriter<K, V> implement
         long bytes = 0;
         for (K key : keys)
             bytes += converter.apply(key).remaining();
-        totalBytes = bytes;
+
+        // an approximation -- the keyset can change while saving
+        estimatedTotalBytes = bytes;
     }
 
     public void saveCache() throws IOException
@@ -67,7 +89,8 @@ public class CacheWriter<K, V> implement
 
     public long getTotalBytes()
     {
-        return totalBytes;
+        // keyset can change in size, thus totalBytes can too
+        return Math.max(estimatedTotalBytes, getBytesComplete());
     }
 
     public long getBytesComplete()

Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java Mon Feb 14 16:06:50 2011
@@ -307,7 +307,7 @@ public class SSTableReader extends SSTab
                     if (shouldAddEntry)
                         indexSummary.addEntry(decoratedKey, indexPosition);
                     if (cacheLoading && keysToLoadInCache.contains(decoratedKey))
-                        keyCache.put(new Pair<Descriptor, DecoratedKey>(descriptor, decoratedKey), dataPosition);
+                        cacheKey(decoratedKey, dataPosition);
                 }
 
                 indexSummary.incrementRowid();
@@ -417,7 +417,9 @@ public class SSTableReader extends SSTab
 
     public void cacheKey(DecoratedKey key, Long info)
     {
-        keyCache.put(new Pair<Descriptor, DecoratedKey>(descriptor, key), info);
+        // avoid keeping a permanent reference to the original key buffer
+        DecoratedKey copiedKey = new DecoratedKey(key.token, key.key == null ? null : ByteBufferUtil.clone(key.key));
+        keyCache.put(new Pair<Descriptor, DecoratedKey>(descriptor, copiedKey), info);
     }
 
     public Long getCachedPosition(DecoratedKey key)
@@ -440,8 +442,12 @@ public class SSTableReader extends SSTab
     public long getPosition(DecoratedKey decoratedKey, Operator op)
     {
         // first, check bloom filter
-        if (op == Operator.EQ && !bf.isPresent(decoratedKey.key))
-            return -1;
+        if (op == Operator.EQ)
+        {
+            assert decoratedKey.key != null; // null is ok for GE scans
+            if (!bf.isPresent(decoratedKey.key))
+                return -1;
+        }
 
         // next, the key cache
         Pair<Descriptor, DecoratedKey> unifiedKey = new Pair<Descriptor, DecoratedKey>(descriptor, decoratedKey);
@@ -481,7 +487,7 @@ public class SSTableReader extends SSTab
                             if (op == Operator.EQ)
                                 bloomFilterTracker.addTruePositive();
                             // store exact match for the key
-                            keyCache.put(unifiedKey, dataPosition);
+                            cacheKey(decoratedKey, dataPosition);
                         }
                         return dataPosition;
                     }

Modified: cassandra/trunk/src/java/org/apache/cassandra/net/MessagingService.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/net/MessagingService.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/net/MessagingService.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/net/MessagingService.java Mon Feb 14 16:06:50 2011
@@ -339,8 +339,8 @@ public final class MessagingService impl
      */
     private void sendOneWay(Message message, String id, InetAddress to)
     {
-        if (logger_.isDebugEnabled())
-            logger_.debug(FBUtilities.getLocalAddress() + " sending " + message.getVerb() + " to " + id + "@" + to);
+        if (logger_.isTraceEnabled())
+            logger_.trace(FBUtilities.getLocalAddress() + " sending " + message.getVerb() + " to " + id + "@" + to);
 
         // do local deliveries
         if ( message.getFrom().equals(to) )

Modified: cassandra/trunk/src/java/org/apache/cassandra/service/AbstractCassandraDaemon.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/service/AbstractCassandraDaemon.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/service/AbstractCassandraDaemon.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/service/AbstractCassandraDaemon.java Mon Feb 14 16:06:50 2011
@@ -161,6 +161,15 @@ public abstract class AbstractCassandraD
             Table.open(table);
         }
 
+        try
+        {
+            GCInspector.instance.start();
+        }
+        catch (Throwable t)
+        {
+            logger.warn("Unable to start GCInspector (currently only supported on the Sun JVM)");
+        }
+
         // replay the log if necessary and check for compaction candidates
         CommitLog.recover();
         CompactionManager.instance.checkAllColumnFamilies();

Modified: cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/service/GCInspector.java Mon Feb 14 16:06:50 2011
@@ -32,6 +32,7 @@ import javax.management.ObjectName;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.utils.StatusLogger;
 
 public class GCInspector
@@ -46,6 +47,7 @@ public class GCInspector
     private HashMap<String, Long> gctimes = new HashMap<String, Long>();
 
     List<Object> beans = new ArrayList<Object>(); // these are instances of com.sun.management.GarbageCollectorMXBean
+    private volatile boolean cacheSizesReduced;
 
     public GCInspector()
     {
@@ -87,13 +89,13 @@ public class GCInspector
         {
             public void run()
             {
-                logIntervalGCStats();
+                logGCResults();
             }
         };
         StorageService.scheduledTasks.scheduleWithFixedDelay(t, INTERVAL_IN_MS, INTERVAL_IN_MS, TimeUnit.MILLISECONDS);
     }
 
-    private void logIntervalGCStats()
+    private void logGCResults()
     {
         for (Object gc : beans)
         {
@@ -121,7 +123,7 @@ public class GCInspector
             }
 
             String st = String.format("GC for %s: %s ms, %s reclaimed leaving %s used; max is %s",
-                    gcw.getName(), gcw.getDuration(), previousMemoryUsed - memoryUsed, memoryUsed, memoryMax);
+                                      gcw.getName(), gcw.getDuration(), previousMemoryUsed - memoryUsed, memoryUsed, memoryMax);
             if (gcw.getDuration() > MIN_DURATION)                          
                 logger.info(st);
             else if (logger.isDebugEnabled())
@@ -129,6 +131,25 @@ public class GCInspector
 
             if (gcw.getDuration() > MIN_DURATION_TPSTATS)
                 StatusLogger.log();
+
+            // if we just finished a full collection and we're still using a lot of memory, try to reduce the pressure
+            if (gcw.getName().equals("ConcurrentMarkSweep"))
+            {
+                double usage = (double) memoryUsed / memoryMax;
+
+                if (memoryUsed > DatabaseDescriptor.getReduceCacheSizesAt() * memoryMax && !cacheSizesReduced)
+                {
+                    cacheSizesReduced = true;
+                    logger.warn("Heap is " + usage + " full.  You may need to reduce memtable and/or cache sizes.  Cassandra is now reducing cache sizes to free up memory.  Adjust reduce_cache_sizes_at threshold in cassandra.yaml if you don't want Cassandra to do this automatically");
+                    StorageService.instance.reduceCacheSizes();
+                }
+
+                if (memoryUsed > DatabaseDescriptor.getFlushLargestMemtablesAt() * memoryMax)
+                {
+                    logger.warn("Heap is " + usage + " full.  You may need to reduce memtable and/or cache sizes.  Cassandra will now flush up to the two largest memtables to free up memory.  Adjust flush_largest_memtables_at threshold in cassandra.yaml if you don't want Cassandra to do this automatically");
+                    StorageService.instance.flushLargestMemtables();
+                }
+            }
         }
     }
 

Modified: cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java Mon Feb 14 16:06:50 2011
@@ -366,15 +366,6 @@ public class StorageService implements I
         initialized = true;
         isClientMode = false;
 
-        try
-        {
-            GCInspector.instance.start();
-        }
-        catch (Throwable t)
-        {
-            logger_.warn("Unable to start GCInspector (currently only supported on the Sun JVM)");
-        }
-
         if (Boolean.parseBoolean(System.getProperty("cassandra.load_ring_state", "true")))
         {
             logger_.info("Loading persisted ring state");
@@ -2177,4 +2168,39 @@ public class StorageService implements I
     {
         return efficientCrossDCWrites;
     }
+
+    /**
+     * Flushes the two largest memtables by ops and by throughput
+     */
+    public void flushLargestMemtables()
+    {
+        ColumnFamilyStore largestByOps = null;
+        ColumnFamilyStore largestByThroughput = null;
+        for (ColumnFamilyStore cfs : ColumnFamilyStore.all())
+        {
+            if (largestByOps == null || cfs.getMemtableColumnsCount() > largestByOps.getMemtableColumnsCount())
+                largestByOps = cfs;
+            if (largestByThroughput == null || cfs.getMemtableThroughputInMB() > largestByThroughput.getMemtableThroughputInMB())
+                largestByThroughput = cfs;
+        }
+        if (largestByOps == null)
+        {
+            logger_.error("Unable to reduce heap usage since there are no column families defined");
+            return;
+        }
+
+        logger_.warn("Flushing " + largestByOps + " to relieve memory pressure");
+        largestByOps.forceFlush();
+        if (largestByThroughput != largestByOps)
+        {
+            logger_.warn("Flushing " + largestByThroughput + " to relieve memory pressure");
+            largestByThroughput.forceFlush();
+        }
+    }
+
+    public void reduceCacheSizes()
+    {
+        for (ColumnFamilyStore cfs : ColumnFamilyStore.all())
+            cfs.reduceCacheSizes();
+    }
 }

Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/NodeCmd.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/NodeCmd.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/tools/NodeCmd.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/tools/NodeCmd.java Mon Feb 14 16:06:50 2011
@@ -458,8 +458,8 @@ public class NodeCmd {
             output.println(String.format("%-10d%10s%18s%18s%18s%18s",
                                          offsets[i],
                                          (i < sprh.length ? sprh[i] : ""),
-                                         (i < rrlh.length ? rrlh[i] : ""),
-                                         (i < rwlh.length ? rwlh[i] : ""),
+                                         (i < rwlh.length ? rrlh[i] : ""),
+                                         (i < rrlh.length ? rwlh[i] : ""),
                                          (i < ersh.length ? ersh[i] : ""),
                                          (i < ecch.length ? ecch[i] : "")));
         }

Modified: cassandra/trunk/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/utils/ByteBufferUtil.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/utils/ByteBufferUtil.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/utils/ByteBufferUtil.java Mon Feb 14 16:06:50 2011
@@ -116,7 +116,7 @@ public class ByteBufferUtil
         ByteBuffer copy = buffer.duplicate();
         copy.position(position);
         copy.limit(copy.position() + length);
-        return string(buffer, charset);
+        return string(copy, charset);
     }
 
     public static String string(ByteBuffer buffer, Charset charset) throws CharacterCodingException
@@ -153,22 +153,19 @@ public class ByteBufferUtil
     }
 
     /**
-     * ByteBuffer adoption of org.apache.commons.lang.ArrayUtils.lastIndexOf method
+     * ByteBuffer adaptation of org.apache.commons.lang.ArrayUtils.lastIndexOf method
      *
      * @param buffer the array to traverse for looking for the object, may be <code>null</code>
      * @param valueToFind the value to find
      * @param startIndex the start index to travers backwards from
-     * @return the last index of the value within the array,
-     * <code>-1</code> if not found or <code>null</code> array input
+     * @return the last index of the value within the array, relative to buffer's arrayOffset
+     * [that is, between buffer.position() and buffer.limit()]; <code>-1</code> if not found.
      */
     public static int lastIndexOf(ByteBuffer buffer, byte valueToFind, int startIndex)
     {
-        if (buffer == null)
-        {
-            return -1;
-        }
+        assert buffer != null;
 
-        if (startIndex < 0)
+        if (startIndex < buffer.position())
         {
             return -1;
         }
@@ -177,12 +174,10 @@ public class ByteBufferUtil
             startIndex = buffer.limit() - 1;
         }
 
-        for (int i = startIndex; i >= 0; i--)
+        for (int i = startIndex; i >= buffer.position(); i--)
         {
             if (valueToFind == buffer.get(i))
-            {
                 return i;
-            }
         }
 
         return -1;

Modified: cassandra/trunk/src/java/org/apache/cassandra/utils/StatusLogger.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/utils/StatusLogger.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/utils/StatusLogger.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/utils/StatusLogger.java Mon Feb 14 16:06:50 2011
@@ -1,4 +1,25 @@
 package org.apache.cassandra.utils;
+/*
+ * 
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * 
+ */
+
 
 import java.lang.management.ManagementFactory;
 import java.util.Set;

Modified: cassandra/trunk/test/unit/org/apache/cassandra/cli/CliTest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/cli/CliTest.java?rev=1070541&r1=1070540&r2=1070541&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/cli/CliTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/cli/CliTest.java Mon Feb 14 16:06:50 2011
@@ -115,6 +115,8 @@ public class CliTest extends CleanupHelp
         "set myCF['key']['scName']['firstname'] = 'John';",
         "get myCF['key']['scName']",
         "use TestKEYSpace;",
+        "describe cluster;",
+        "help describe cluster;",
         "show cluster name",
         "show api version",
         "help help",



Mime
View raw message