cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject svn commit: r923410 - in /cassandra/trunk: ./ conf/ interface/thrift/gen-java/org/apache/cassandra/thrift/ src/java/org/apache/cassandra/config/ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/db/filter/ src/java/org/apache/cassandra/se...
Date Mon, 15 Mar 2010 19:53:05 GMT
Author: jbellis
Date: Mon Mar 15 19:53:05 2010
New Revision: 923410

URL: http://svn.apache.org/viewvc?rev=923410&view=rev
Log:
merge from 0.6

Modified:
    cassandra/trunk/   (props changed)
    cassandra/trunk/CHANGES.txt
    cassandra/trunk/conf/storage-conf.xml
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java 
 (props changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java   (props
changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
  (props changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
  (props changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
  (props changed)
    cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java
    cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
    cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
    cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
    cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java
    cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java
    cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java
    cassandra/trunk/src/java/org/apache/cassandra/service/StorageServiceMBean.java

Propchange: cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Mar 15 19:53:05 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6:922689-923252
+/cassandra/branches/cassandra-0.6:922689-923409
 /incubator/cassandra/branches/cassandra-0.3:774578-796573
 /incubator/cassandra/branches/cassandra-0.4:810145-834239,834349-834350
 /incubator/cassandra/branches/cassandra-0.5:888872-915439

Modified: cassandra/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=923410&r1=923409&r2=923410&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Mon Mar 15 19:53:05 2010
@@ -30,6 +30,11 @@ dev
    since those settings are no longer used (CASSANDRA-878)
  * Parse a long from RowWarningThresholdInMB instead of an int (CASSANDRA-882)
  * Remove obsolete ControlPort code from DatabaseDescriptor (CASSANDRA-886)
+ * move skipBytes side effect out of assert (CASSANDRA-899)
+ * add "double getLoad" to StorageServiceMBean (CASSANDRA-898)
+ * track row stats per CF at compaction time (CASSANDRA-870)
+ * disallow CommitLogDirectory matching a DataFileDirectory (CASSANDRA-888)
+ * default key cache size is 200k entries, changed from 10% (CASSANDRA-863)
 
 
 0.6.0-beta1/beta2

Modified: cassandra/trunk/conf/storage-conf.xml
URL: http://svn.apache.org/viewvc/cassandra/trunk/conf/storage-conf.xml?rev=923410&r1=923409&r2=923410&view=diff
==============================================================================
--- cassandra/trunk/conf/storage-conf.xml (original)
+++ cassandra/trunk/conf/storage-conf.xml Mon Mar 15 19:53:05 2010
@@ -87,25 +87,33 @@
        ~ the number of keys per sstable whose locations we keep in
        ~ memory in "mostly LRU" order.  (JUST the key locations, NOT any
        ~ column values.) Specify a fraction (value less than 1), a percentage
-       ~ (ending in a % sign) or an absolute number of keys to cache. 
+       ~ (ending in a % sign) or an absolute number of keys to cache.
+       ~ KeysCached defaults to 200000 keys.
        ~
        ~ The optional RowsCached attribute specifies the number of rows
        ~ whose entire contents we cache in memory. Do not use this on
        ~ ColumnFamilies with large rows, or ColumnFamilies with high write:read
        ~ ratios. Specify a fraction (value less than 1), a percentage (ending in
        ~ a % sign) or an absolute number of rows to cache. 
+       ~ RowsCached defaults to 0, i.e., row cache is off by default.
+       ~
+       ~ Remember, when using caches as a percentage, they WILL grow with
+       ~ your data set!
       -->
-      <ColumnFamily CompareWith="BytesType" 
-                    Name="Standard1" 
-                    RowsCached="10%"
-                    KeysCached="0"/>
-      <ColumnFamily CompareWith="UTF8Type" Name="Standard2"/>
-      <ColumnFamily CompareWith="TimeUUIDType" Name="StandardByUUID1"/>
-      <ColumnFamily ColumnType="Super"
+      <ColumnFamily Name="Standard1" CompareWith="BytesType"/>
+      <ColumnFamily Name="Standard2" 
+                    CompareWith="UTF8Type"
+                    KeysCached="100%"/>
+      <ColumnFamily Name="StandardByUUID1" CompareWith="TimeUUIDType" />
+      <ColumnFamily Name="Super1"
+                    ColumnType="Super"
+                    CompareWith="BytesType"
+                    CompareSubcolumnsWith="BytesType" />
+      <ColumnFamily Name="Super2"
+                    ColumnType="Super"
                     CompareWith="UTF8Type"
                     CompareSubcolumnsWith="UTF8Type"
-                    Name="Super1"
-                    RowsCached="1000"
+                    RowsCached="10000"
                     KeysCached="50%"
                     Comment="A column family with supercolumns, whose column and subcolumn
names are UTF8 strings"/>
 
@@ -341,7 +349,7 @@
    ~ This is less necessary in Cassandra than in traditional databases
    ~ since replication reduces the odds of losing data from a failure
    ~ after writing the log entry but before it actually reaches the disk.
-   ~ So the other option is "timed," where writes may be acked immediately
+   ~ So the other option is "periodic," where writes may be acked immediately
    ~ and the CommitLog is simply synced every CommitLogSyncPeriodInMS
    ~ milliseconds.
   -->

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Mar 15 19:53:05 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-923252
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-923409
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573
 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Cassandra.java:810145-834239,834349-834350
 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Cassandra.java:888872-903502

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Mar 15 19:53:05 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-923252
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-923409
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198
 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Column.java:810145-834239,834349-834350
 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/Column.java:888872-903502

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Mar 15 19:53:05 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-923252
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-923409
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-796573
 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:810145-834239,834349-834350
 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:888872-903502

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Mar 15 19:53:05 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-923252
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-923409
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-796573
 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:810145-834239,834349-834350
 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:888872-903502

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Mar 15 19:53:05 2010
@@ -1,4 +1,4 @@
-/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-923252
+/cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-923409
 /incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-792198
 /incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:810145-834239,834349-834350
 /incubator/cassandra/branches/cassandra-0.5/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:888872-903502

Modified: cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java?rev=923410&r1=923409&r2=923410&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java Mon Mar 15 19:53:05
2010
@@ -40,7 +40,7 @@ import org.apache.commons.lang.ObjectUti
 
 public final class CFMetaData
 {
-    public final static double DEFAULT_KEY_CACHE_SIZE = 0.1;
+    public final static double DEFAULT_KEY_CACHE_SIZE = 200000;
     public final static double DEFAULT_ROW_CACHE_SIZE = 0.0;
 
     private static final AtomicInteger idGen = new AtomicInteger(0);

Modified: cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=923410&r1=923409&r2=923410&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java Mon Mar 15
19:53:05 2010
@@ -447,6 +447,12 @@ public class DatabaseDescriptor
             dataFileDirectories = xmlUtils.getNodeValues("/Storage/DataFileDirectories/DataFileDirectory");
             logFileDirectory = xmlUtils.getNodeValue("/Storage/CommitLogDirectory");
 
+            for (String datadir : dataFileDirectories)
+            {
+                if (datadir.equals(logFileDirectory))
+                    throw new ConfigurationException("CommitLogDirectory must not be the
same as any DataFileDirectory");
+            }
+
             /* threshold after which commit log should be rotated. */
             String value = xmlUtils.getNodeValue("/Storage/CommitLogRotationThresholdInMB");
             if ( value != null)

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=923410&r1=923409&r2=923410&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Mon Mar 15 19:53:05
2010
@@ -119,6 +119,11 @@ public class ColumnFamilyStore implement
 
     private LatencyTracker readStats_ = new LatencyTracker();
     private LatencyTracker writeStats_ = new LatencyTracker();
+
+    private long minRowCompactedSize = 0L;
+    private long maxRowCompactedSize = 0L;
+    private long rowsCompactedTotalSize = 0L;
+    private long rowsCompactedCount = 0L;
     
     ColumnFamilyStore(String table, String columnFamilyName, boolean isSuper, int indexValue)
throws IOException
     {
@@ -188,6 +193,34 @@ public class ColumnFamilyStore implement
         ssTables_.add(sstables);
     }
 
+    public void addToCompactedRowStats(Long rowsize)
+    {
+        if (minRowCompactedSize < 1 || rowsize < minRowCompactedSize)
+            minRowCompactedSize = rowsize;
+        if (rowsize > maxRowCompactedSize)
+            maxRowCompactedSize = rowsize;
+        rowsCompactedCount++;
+        rowsCompactedTotalSize += rowsize;
+    }
+
+    public long getMinRowCompactedSize()
+    {
+        return minRowCompactedSize;
+    }
+
+    public long getMaxRowCompactedSize()
+    {
+        return maxRowCompactedSize;
+    }
+
+    public long getMeanRowCompactedSize()
+    {
+        if (rowsCompactedCount > 0)
+            return rowsCompactedTotalSize / rowsCompactedCount;
+        else
+            return 0L;
+    }
+
     public static ColumnFamilyStore createColumnFamilyStore(String table, String columnFamily)
throws IOException
     {
         /*

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java?rev=923410&r1=923409&r2=923410&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java Mon Mar 15
19:53:05 2010
@@ -118,4 +118,22 @@ public interface ColumnFamilyStoreMBean
      * invalidate the row cache; for use after bulk loading via BinaryMemtable
      */
     public void invalidateRowCache();
+
+    /**
+     * return the size of the smallest compacted row
+     * @return
+     */
+    public long getMinRowCompactedSize();
+
+    /**
+     * return the size of the largest compacted row
+     * @return
+     */
+    public long getMaxRowCompactedSize();
+
+    /**
+     * return the mean size of the rows compacted
+     * @return
+     */
+    public long getMeanRowCompactedSize();
 }

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java?rev=923410&r1=923409&r2=923410&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java Mon Mar 15 19:53:05
2010
@@ -359,6 +359,7 @@ public class CompactionManager implement
                 long rowsize = writer.getFilePointer() - prevpos;
                 if (rowsize > DatabaseDescriptor.getRowWarningThreshold())
                     logger.warn("Large row " + row.key.key + " in " + cfs.getColumnFamilyName()
+ " " + rowsize + " bytes");
+                cfs.addToCompactedRowStats(rowsize);
             }
             validator.complete();
         }

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java?rev=923410&r1=923409&r2=923410&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java Mon
Mar 15 19:53:05 2010
@@ -95,7 +95,8 @@ public class SSTableNamesIterator extend
             for (IndexHelper.IndexInfo indexInfo : ranges)
             {
                 file.reset();
-                assert file.skipBytes((int)indexInfo.offset) == indexInfo.offset;
+                long curOffsert = file.skipBytes((int)indexInfo.offset);
+                assert curOffsert == indexInfo.offset;
                 // TODO only completely deserialize columns we are interested in
                 while (file.bytesPastMark() < indexInfo.offset + indexInfo.width)
                 {

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java?rev=923410&r1=923409&r2=923410&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java Mon
Mar 15 19:53:05 2010
@@ -200,7 +200,8 @@ class SSTableSliceIterator extends Abstr
             boolean outOfBounds = false;
 
             file.reset();
-            assert file.skipBytes((int)curColPosition.offset) == curColPosition.offset;
+            long curOffset = file.skipBytes((int) curColPosition.offset); 
+            assert curOffset == curColPosition.offset;
             while (file.bytesPastMark() < curColPosition.offset + curColPosition.width
&& !outOfBounds)
             {
                 IColumn column = emptyColumnFamily.getColumnSerializer().deserialize(file);

Modified: cassandra/trunk/src/java/org/apache/cassandra/service/StorageServiceMBean.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/service/StorageServiceMBean.java?rev=923410&r1=923409&r2=923410&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/service/StorageServiceMBean.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/service/StorageServiceMBean.java Mon Mar
15 19:53:05 2010
@@ -62,6 +62,11 @@ public interface StorageServiceMBean
      */
     public Map<Range, List<String>> getRangeToEndPointMap(String keyspace);
 
+    /**
+     * Numeric load value.
+     */
+    public double getLoad();
+
     /** Human-readable load value */
     public String getLoadString();
 



Mime
View raw message