cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From brandonwilli...@apache.org
Subject svn commit: r992524 - in /cassandra/trunk: src/java/org/apache/cassandra/config/ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/io/sstable/ src/java/org/apache/cassandra/utils/ test/unit/org/apache/cassandra/db/
Date Sat, 04 Sep 2010 03:31:38 GMT
Author: brandonwilliams
Date: Sat Sep  4 03:31:36 2010
New Revision: 992524

URL: http://svn.apache.org/viewvc?rev=992524&view=rev
Log:
Move persistent sstable stats from the system table to an sstable component.  Patch by brandonwilliams,
reviewed by jbellis for CASSANDRA-1430

Removed:
    cassandra/trunk/src/java/org/apache/cassandra/db/StatisticsTable.java
Modified:
    cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java
    cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
    cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
    cassandra/trunk/src/java/org/apache/cassandra/db/Table.java
    cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java
    cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java
    cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
    cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
    cassandra/trunk/src/java/org/apache/cassandra/utils/EstimatedHistogram.java
    cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java
    cassandra/trunk/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java

Modified: cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java Sat Sep  4 03:31:36
2010
@@ -39,7 +39,6 @@ import org.apache.cassandra.db.clock.Abs
 import org.apache.cassandra.db.clock.TimestampReconciler;
 import org.apache.cassandra.db.HintedHandOffManager;
 import org.apache.cassandra.db.SystemTable;
-import org.apache.cassandra.db.StatisticsTable;
 import org.apache.cassandra.db.Table;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.BytesType;
@@ -65,7 +64,6 @@ public final class CFMetaData
     public static final CFMetaData HintsCf = newSystemTable(HintedHandOffManager.HINTS_CF,
1, "hinted handoff data", BytesType.instance, BytesType.instance);
     public static final CFMetaData MigrationsCf = newSystemTable(Migration.MIGRATIONS_CF,
2, "individual schema mutations", TimeUUIDType.instance, null);
     public static final CFMetaData SchemaCf = newSystemTable(Migration.SCHEMA_CF, 3, "current
state of the schema", UTF8Type.instance, null);
-    public static final CFMetaData StatisticsCf = newSystemTable(StatisticsTable.STATISTICS_CF,
4, "persistent CF statistics for the local node", UTF8Type.instance, BytesType.instance);
 
     private static CFMetaData newSystemTable(String cfName, int cfId, String comment, AbstractType
comparator, AbstractType subComparator)
     {

Modified: cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java Sat Sep 
4 03:31:36 2010
@@ -355,13 +355,11 @@ public class DatabaseDescriptor
                                                                     CFMetaData.HintsCf,
                                                                     CFMetaData.MigrationsCf,
                                                                     CFMetaData.SchemaCf,
-                                                                    CFMetaData.StatisticsCf
             });
             CFMetaData.map(CFMetaData.StatusCf);
             CFMetaData.map(CFMetaData.HintsCf);
             CFMetaData.map(CFMetaData.MigrationsCf);
             CFMetaData.map(CFMetaData.SchemaCf);
-            CFMetaData.map(CFMetaData.StatisticsCf);
             tables.put(Table.SYSTEM_TABLE, systemMeta);
             
             /* Load the seeds for node contact points */

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Sat Sep  4 03:31:36
2010
@@ -382,37 +382,6 @@ public class ColumnFamilyStore implement
         }
     }
     
-    // returns runnables that need to update the system table.
-    static Collection<Runnable> deleteCompactedFiles(String table, String columnFamily)
-    {
-        Collection<Runnable> runnables = new ArrayList<Runnable>();
-        for (File file : files(table, columnFamily))
-        {
-            if (file.getName().contains("-Data.db"))
-            {
-                final String delPath = file.getAbsolutePath();
-                if (SSTable.deleteIfCompacted(delPath))
-                {
-                    runnables.add(new Runnable()
-                    {
-                        public void run()
-                        {
-                            try
-                            {
-                                StatisticsTable.deleteSSTableStatistics(delPath);
-                            }
-                            catch (IOException ex)
-                            {
-                                throw new RuntimeException(ex);
-                            }
-                        }
-                    });
-                }
-            }
-        }
-        return runnables;
-    }
-
     private static Set<File> files(String table, String columnFamily)
     {
         assert table != null;

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/Table.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/Table.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/Table.java Sat Sep  4 03:31:36 2010
@@ -97,17 +97,12 @@ public class Table
                 if (tableInstance == null)
                 {
                     // do some housekeeping on the column families.
-                    Collection<Runnable> systemTableUpdates = new ArrayList<Runnable>();
                     for (CFMetaData cfm : DatabaseDescriptor.getTableDefinition(table).cfMetaData().values())
                     {
                         ColumnFamilyStore.scrubDataDirectories(table, cfm.cfName);
-                        systemTableUpdates.addAll(ColumnFamilyStore.deleteCompactedFiles(table,
cfm.cfName)); 
                     }
                     tableInstance = new Table(table);
                     instances.put(table, tableInstance);
-                    
-                    for (Runnable r : systemTableUpdates)
-                        r.run();
                 }
             }
         }

Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java Sat Sep  4 03:31:36
2010
@@ -32,7 +32,6 @@ import org.slf4j.LoggerFactory;
 import org.apache.cassandra.config.CFMetaData;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.db.StatisticsTable;
 import org.apache.cassandra.utils.EstimatedHistogram;
 
 /**
@@ -54,6 +53,7 @@ public abstract class SSTable
     public static final String COMPONENT_DATA = "Data.db";
     public static final String COMPONENT_INDEX = "Index.db";
     public static final String COMPONENT_FILTER = "Filter.db";
+    public static final String COMPONENT_STATS = "Statistics.db";
 
     public static final String COMPONENT_COMPACTED = "Compacted";
 
@@ -63,7 +63,7 @@ public abstract class SSTable
 
     public static final String TEMPFILE_MARKER = "tmp";
 
-    public static List<String> components = Collections.unmodifiableList(Arrays.asList(COMPONENT_FILTER,
COMPONENT_INDEX, COMPONENT_DATA));
+    public static List<String> components = Collections.unmodifiableList(Arrays.asList(COMPONENT_FILTER,
COMPONENT_INDEX, COMPONENT_DATA, COMPONENT_STATS));
     protected EstimatedHistogram estimatedRowSize = new EstimatedHistogram(150);
     protected EstimatedHistogram estimatedColumnCount = new EstimatedHistogram(114);
 
@@ -132,6 +132,7 @@ public abstract class SSTable
                 FileUtils.deleteWithConfirm(new File(dataFilename));
                 FileUtils.deleteWithConfirm(new File(SSTable.indexFilename(dataFilename)));
                 FileUtils.deleteWithConfirm(new File(SSTable.filterFilename(dataFilename)));
+                FileUtils.deleteWithConfirm(new File(SSTable.statisticsFilename(dataFilename)));
                 FileUtils.deleteWithConfirm(new File(SSTable.compactedFilename(dataFilename)));
             }
             catch (IOException e)
@@ -154,11 +155,21 @@ public abstract class SSTable
         return Descriptor.fromFilename(dataFile).filenameFor(COMPONENT_FILTER);
     }
 
+    protected static String statisticsFilename(String dataFile)
+    {
+        return Descriptor.fromFilename(dataFile).filenameFor(COMPONENT_STATS);
+    }
+
     public String filterFilename()
     {
         return desc.filenameFor(COMPONENT_FILTER);
     }
 
+    public String statisticsFilename()
+    {
+        return desc.filenameFor(COMPONENT_STATS);
+    }
+
     public String getFilename()
     {
         return desc.filenameFor(COMPONENT_DATA);

Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java
(original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java
Sat Sep  4 03:31:36 2010
@@ -30,7 +30,6 @@ import java.util.TimerTask;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.db.StatisticsTable;
 import org.apache.cassandra.io.DeletionService;
 import org.apache.cassandra.io.util.FileUtils;
 
@@ -95,8 +94,8 @@ public class SSTableDeletingReference ex
             {
                 FileUtils.deleteWithConfirm(new File(SSTable.indexFilename(path)));
                 FileUtils.deleteWithConfirm(new File(SSTable.filterFilename(path)));
+                FileUtils.deleteWithConfirm(new File(SSTable.statisticsFilename(path)));
                 FileUtils.deleteWithConfirm(new File(SSTable.compactedFilename(path)));
-                StatisticsTable.deleteSSTableStatistics(datafile.getAbsolutePath());
             }
             catch (IOException e)
             {

Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java Sat Sep  4
03:31:36 2010
@@ -143,18 +143,14 @@ public class SSTableReader extends SSTab
 
     private void loadStatistics(Descriptor desc) throws IOException
     {
-        // skip loading stats for the system table, or we will infinitely recurse
-        if (desc.ksname.equals(Table.SYSTEM_TABLE))
+        if (!new File(desc.filenameFor(SSTable.COMPONENT_STATS)).exists())
             return;
         if (logger.isDebugEnabled())
             logger.debug("Load statistics for " + desc);
-        long[] rowsizes = StatisticsTable.getSSTableRowSizeStatistics(desc.filenameFor(SSTable.COMPONENT_DATA));
-        long[] colcounts = StatisticsTable.getSSTableColumnCountStatistics(desc.filenameFor(SSTable.COMPONENT_DATA));
-        if (rowsizes.length > 0)
-        {
-            estimatedRowSize = new EstimatedHistogram(rowsizes);
-            estimatedColumnCount = new EstimatedHistogram(colcounts);
-        }
+        DataInputStream dis = new DataInputStream(new FileInputStream(desc.filenameFor(SSTable.COMPONENT_STATS)));
+        estimatedRowSize = EstimatedHistogram.serializer.deserialize(dis);
+        estimatedColumnCount = EstimatedHistogram.serializer.deserialize(dis);
+        dis.close();
     }
 
     public static SSTableReader open(Descriptor desc) throws IOException

Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java Sat Sep  4
03:31:36 2010
@@ -37,7 +37,7 @@ import org.apache.cassandra.io.util.Segm
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.BloomFilter;
 import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.WrappedRunnable;
+import org.apache.cassandra.utils.EstimatedHistogram;
 
 public class SSTableWriter extends SSTable
 {
@@ -140,17 +140,12 @@ public class SSTableWriter extends SSTab
         // main data
         dataFile.close(); // calls force
 
+        // write sstable statistics
+        writeStatistics(desc);
+
         // remove the 'tmp' marker from all components
         final Descriptor newdesc = rename(desc);
 
-        Runnable runnable = new WrappedRunnable()
-        {
-            protected void runMayThrow() throws IOException
-            {
-                StatisticsTable.persistSSTableStatistics(newdesc, estimatedRowSize, estimatedColumnCount);
-            }
-        };
-        ColumnFamilyStore.submitPostFlush(runnable);
 
         // finalize in-memory state for the reader
         SegmentedFile ifile = iwriter.builder.complete(newdesc.filenameFor(SSTable.COMPONENT_INDEX));
@@ -161,6 +156,14 @@ public class SSTableWriter extends SSTab
         return sstable;
     }
 
+    private void writeStatistics(Descriptor desc) throws IOException
+    {
+        DataOutputStream dos = new DataOutputStream(new FileOutputStream(desc.filenameFor(SSTable.COMPONENT_STATS)));
+        EstimatedHistogram.serializer.serialize(estimatedRowSize, dos);
+        EstimatedHistogram.serializer.serialize(estimatedColumnCount, dos);
+        dos.close();
+    }
+
     static Descriptor rename(Descriptor tmpdesc)
     {
         Descriptor newdesc = tmpdesc.asTemporary(false);

Modified: cassandra/trunk/src/java/org/apache/cassandra/utils/EstimatedHistogram.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/utils/EstimatedHistogram.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/utils/EstimatedHistogram.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/utils/EstimatedHistogram.java Sat Sep  4
03:31:36 2010
@@ -18,8 +18,13 @@
 */
 package org.apache.cassandra.utils;
 
+import java.io.IOException;
 import java.util.concurrent.atomic.AtomicLongArray;
 import java.util.Arrays;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+
+import org.apache.cassandra.io.ICompactSerializer;
 
 public class EstimatedHistogram
 {
@@ -38,6 +43,8 @@ public class EstimatedHistogram
 
     final AtomicLongArray buckets;
 
+    public static EstimatedHistogramSerializer serializer = new EstimatedHistogramSerializer();
+
     public EstimatedHistogram()
     {
         this(90);
@@ -55,6 +62,14 @@ public class EstimatedHistogram
         buckets = new AtomicLongArray(bucketData);
     }
 
+    public EstimatedHistogram(long[] offsets, long[] bucketData)
+    {
+        assert bucketData.length == offsets.length +1;
+        bucketOffsets = offsets;
+        buckets = new AtomicLongArray(bucketData);
+        numBuckets = bucketData.length;
+    }
+
     private void makeOffsets(int size)
     {
         bucketOffsets = new long[size];
@@ -144,4 +159,32 @@ public class EstimatedHistogram
         }
         return median;
     }
+
+    public static class EstimatedHistogramSerializer implements ICompactSerializer<EstimatedHistogram>
+    {
+        public void serialize(EstimatedHistogram eh, DataOutputStream dos) throws IOException
+        {
+            long[] offsets = eh.getBucketOffsets();
+            long[] buckets = eh.get(false);
+            dos.writeInt(buckets.length);
+            for (int i = 0; i < buckets.length; i++)
+            {
+                dos.writeLong(offsets[i == 0 ? 0 : i - 1]);
+                dos.writeLong(buckets[i]);
+            }
+        }
+
+        public EstimatedHistogram deserialize(DataInputStream dis) throws IOException
+        {
+            int size = dis.readInt();
+            long[] offsets = new long[size - 1];
+            long[] buckets = new long[size];
+
+            for (int i = 0; i < size; i++) {
+                offsets[i == 0 ? 0 : i - 1] = dis.readLong();
+                buckets[i] = dis.readLong();
+            }
+            return new EstimatedHistogram(offsets, buckets);
+        }
+    }
 }

Modified: cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java Sat Sep  4 03:31:36 2010
@@ -51,7 +51,6 @@ public class DefsTest extends CleanupHel
         assert CFMetaData.HintsCf.cfId == 1;    
         assert CFMetaData.MigrationsCf.cfId == 2;    
         assert CFMetaData.SchemaCf.cfId == 3;    
-        assert CFMetaData.StatisticsCf.cfId == 4;    
     }
     
     @Test

Modified: cassandra/trunk/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java?rev=992524&r1=992523&r2=992524&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java Sat Sep  4
03:31:36 2010
@@ -48,11 +48,6 @@ public class RecoveryManager2Test extend
         ColumnFamilyStore cfs = table1.getColumnFamilyStore("Standard1");
         cfs.forceBlockingFlush();
 
-        // forceBlockingFlush above adds persistent stats to the current commit log segment
-        // it ends up in the same segment as key99 meaning that segment still has unwritten
data
-        // thus the commit log replays it when recover is called below
-        Table.open(Table.SYSTEM_TABLE).getColumnFamilyStore(StatisticsTable.STATISTICS_CF).forceBlockingFlush();
-
         // remove all SSTable/MemTables
         cfs.clearUnsafe();
 



Mime
View raw message