cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject svn commit: r918524 - in /incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra: config/DatabaseDescriptor.java db/CompactionManager.java io/SSTableWriter.java
Date Wed, 03 Mar 2010 15:43:20 GMT
Author: jbellis
Date: Wed Mar  3 15:43:20 2010
New Revision: 918524

URL: http://svn.apache.org/viewvc?rev=918524&view=rev
Log:
warn when exceptionally large rows are seen during compaction.  patch by Brandon Williams;
reviewed by jbellis for CASSANDRA-843

Modified:
    incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
    incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/db/CompactionManager.java
    incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableWriter.java

Modified: incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=918524&r1=918523&r2=918524&view=diff
==============================================================================
--- incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
(original)
+++ incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
Wed Mar  3 15:43:20 2010
@@ -91,6 +91,8 @@
 
     static Map<String, KSMetaData> tables = new HashMap<String, KSMetaData>();
     private static int bmtThreshold = 256;
+    /* if this a row exceeds this threshold, we issue warnings during compaction */
+    private static long rowWarningThreshold = 512 * 1024 * 1024;
 
     /* Hashing strategy Random or OPHF */
     private static IPartitioner partitioner;
@@ -436,6 +438,13 @@
                 columnIndexSizeInKB = Integer.parseInt(columnIndexSize);
             }
 
+            String rowWarning = xmlUtils.getNodeValue("/Storage/RowWarningThresholdInMB");
+            if (rowWarning != null)
+            {
+                rowWarningThreshold = Integer.parseInt(rowWarning) * 1024 * 1024;
+                if (rowWarningThreshold <= 0)
+                    throw new ConfigurationException("Row warning threshold must be a positive
integer");
+            }
             /* data file and commit log directories. they get created later, when they're
needed. */
             dataFileDirectories = xmlUtils.getNodeValues("/Storage/DataFileDirectories/DataFileDirectory");
             logFileDirectory = xmlUtils.getNodeValue("/Storage/CommitLogDirectory");
@@ -982,6 +991,11 @@
         return concurrentWriters;
     }
 
+    public static long getRowWarningThreshold()
+    {
+        return rowWarningThreshold;
+    }
+    
     public static String[] getAllDataFileLocations()
     {
         return dataFileDirectories;

Modified: incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/db/CompactionManager.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/db/CompactionManager.java?rev=918524&r1=918523&r2=918524&view=diff
==============================================================================
--- incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/db/CompactionManager.java
(original)
+++ incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/db/CompactionManager.java
Wed Mar  3 15:43:20 2010
@@ -299,9 +299,15 @@
             while (nni.hasNext())
             {
                 CompactionIterator.CompactedRow row = nni.next();
+                long prevpos = writer.getFilePointer();
+
                 writer.append(row.key, row.buffer);
                 validator.add(row);
                 totalkeysWritten++;
+
+                long rowsize = writer.getFilePointer() - prevpos;
+                if (rowsize > DatabaseDescriptor.getRowWarningThreshold())
+                    logger.warn("Large row " + row.key.key + " in " + cfs.getColumnFamilyName()
+ " " + rowsize + " bytes");
             }
             validator.complete();
         }

Modified: incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableWriter.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableWriter.java?rev=918524&r1=918523&r2=918524&view=diff
==============================================================================
--- incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableWriter.java
(original)
+++ incubator/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableWriter.java
Wed Mar  3 15:43:20 2010
@@ -170,6 +170,11 @@
         return filename;
     }
 
+    public long getFilePointer()
+    {
+        return dataFile.getFilePointer();
+    }
+    
     public static SSTableReader renameAndOpen(String dataFileName) throws IOException
     {
         SSTableWriter.rename(indexFilename(dataFileName));



Mime
View raw message