Return-Path: Delivered-To: apmail-cassandra-commits-archive@www.apache.org Received: (qmail 50331 invoked from network); 4 Sep 2010 03:32:04 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 4 Sep 2010 03:32:04 -0000 Received: (qmail 96031 invoked by uid 500); 4 Sep 2010 03:32:04 -0000 Delivered-To: apmail-cassandra-commits-archive@cassandra.apache.org Received: (qmail 95890 invoked by uid 500); 4 Sep 2010 03:32:01 -0000 Mailing-List: contact commits-help@cassandra.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cassandra.apache.org Delivered-To: mailing list commits@cassandra.apache.org Received: (qmail 95881 invoked by uid 99); 4 Sep 2010 03:32:00 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 04 Sep 2010 03:32:00 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 04 Sep 2010 03:31:58 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 8775823889C5; Sat, 4 Sep 2010 03:31:38 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r992524 - in /cassandra/trunk: src/java/org/apache/cassandra/config/ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/io/sstable/ src/java/org/apache/cassandra/utils/ test/unit/org/apache/cassandra/db/ Date: Sat, 04 Sep 2010 03:31:38 -0000 To: commits@cassandra.apache.org From: brandonwilliams@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20100904033138.8775823889C5@eris.apache.org> Author: brandonwilliams Date: Sat Sep 4 03:31:36 2010 New Revision: 992524 URL: http://svn.apache.org/viewvc?rev=992524&view=rev Log: Move persistent sstable stats from the system table to an sstable component. Patch by brandonwilliams, reviewed by jbellis for CASSANDRA-1430 Removed: cassandra/trunk/src/java/org/apache/cassandra/db/StatisticsTable.java Modified: cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java cassandra/trunk/src/java/org/apache/cassandra/db/Table.java cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java cassandra/trunk/src/java/org/apache/cassandra/utils/EstimatedHistogram.java cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java cassandra/trunk/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java Modified: cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java?rev=992524&r1=992523&r2=992524&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java Sat Sep 4 03:31:36 2010 @@ -39,7 +39,6 @@ import org.apache.cassandra.db.clock.Abs import org.apache.cassandra.db.clock.TimestampReconciler; import org.apache.cassandra.db.HintedHandOffManager; import org.apache.cassandra.db.SystemTable; -import org.apache.cassandra.db.StatisticsTable; import org.apache.cassandra.db.Table; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.BytesType; @@ -65,7 +64,6 @@ public final class CFMetaData public static final CFMetaData HintsCf = newSystemTable(HintedHandOffManager.HINTS_CF, 1, "hinted handoff data", BytesType.instance, BytesType.instance); public static final CFMetaData MigrationsCf = newSystemTable(Migration.MIGRATIONS_CF, 2, "individual schema mutations", TimeUUIDType.instance, null); public static final CFMetaData SchemaCf = newSystemTable(Migration.SCHEMA_CF, 3, "current state of the schema", UTF8Type.instance, null); - public static final CFMetaData StatisticsCf = newSystemTable(StatisticsTable.STATISTICS_CF, 4, "persistent CF statistics for the local node", UTF8Type.instance, BytesType.instance); private static CFMetaData newSystemTable(String cfName, int cfId, String comment, AbstractType comparator, AbstractType subComparator) { Modified: cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=992524&r1=992523&r2=992524&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java Sat Sep 4 03:31:36 2010 @@ -355,13 +355,11 @@ public class DatabaseDescriptor CFMetaData.HintsCf, CFMetaData.MigrationsCf, CFMetaData.SchemaCf, - CFMetaData.StatisticsCf }); CFMetaData.map(CFMetaData.StatusCf); CFMetaData.map(CFMetaData.HintsCf); CFMetaData.map(CFMetaData.MigrationsCf); CFMetaData.map(CFMetaData.SchemaCf); - CFMetaData.map(CFMetaData.StatisticsCf); tables.put(Table.SYSTEM_TABLE, systemMeta); /* Load the seeds for node contact points */ Modified: cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=992524&r1=992523&r2=992524&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Sat Sep 4 03:31:36 2010 @@ -382,37 +382,6 @@ public class ColumnFamilyStore implement } } - // returns runnables that need to update the system table. - static Collection deleteCompactedFiles(String table, String columnFamily) - { - Collection runnables = new ArrayList(); - for (File file : files(table, columnFamily)) - { - if (file.getName().contains("-Data.db")) - { - final String delPath = file.getAbsolutePath(); - if (SSTable.deleteIfCompacted(delPath)) - { - runnables.add(new Runnable() - { - public void run() - { - try - { - StatisticsTable.deleteSSTableStatistics(delPath); - } - catch (IOException ex) - { - throw new RuntimeException(ex); - } - } - }); - } - } - } - return runnables; - } - private static Set files(String table, String columnFamily) { assert table != null; Modified: cassandra/trunk/src/java/org/apache/cassandra/db/Table.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java?rev=992524&r1=992523&r2=992524&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/db/Table.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/db/Table.java Sat Sep 4 03:31:36 2010 @@ -97,17 +97,12 @@ public class Table if (tableInstance == null) { // do some housekeeping on the column families. - Collection systemTableUpdates = new ArrayList(); for (CFMetaData cfm : DatabaseDescriptor.getTableDefinition(table).cfMetaData().values()) { ColumnFamilyStore.scrubDataDirectories(table, cfm.cfName); - systemTableUpdates.addAll(ColumnFamilyStore.deleteCompactedFiles(table, cfm.cfName)); } tableInstance = new Table(table); instances.put(table, tableInstance); - - for (Runnable r : systemTableUpdates) - r.run(); } } } Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=992524&r1=992523&r2=992524&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java Sat Sep 4 03:31:36 2010 @@ -32,7 +32,6 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.db.StatisticsTable; import org.apache.cassandra.utils.EstimatedHistogram; /** @@ -54,6 +53,7 @@ public abstract class SSTable public static final String COMPONENT_DATA = "Data.db"; public static final String COMPONENT_INDEX = "Index.db"; public static final String COMPONENT_FILTER = "Filter.db"; + public static final String COMPONENT_STATS = "Statistics.db"; public static final String COMPONENT_COMPACTED = "Compacted"; @@ -63,7 +63,7 @@ public abstract class SSTable public static final String TEMPFILE_MARKER = "tmp"; - public static List components = Collections.unmodifiableList(Arrays.asList(COMPONENT_FILTER, COMPONENT_INDEX, COMPONENT_DATA)); + public static List components = Collections.unmodifiableList(Arrays.asList(COMPONENT_FILTER, COMPONENT_INDEX, COMPONENT_DATA, COMPONENT_STATS)); protected EstimatedHistogram estimatedRowSize = new EstimatedHistogram(150); protected EstimatedHistogram estimatedColumnCount = new EstimatedHistogram(114); @@ -132,6 +132,7 @@ public abstract class SSTable FileUtils.deleteWithConfirm(new File(dataFilename)); FileUtils.deleteWithConfirm(new File(SSTable.indexFilename(dataFilename))); FileUtils.deleteWithConfirm(new File(SSTable.filterFilename(dataFilename))); + FileUtils.deleteWithConfirm(new File(SSTable.statisticsFilename(dataFilename))); FileUtils.deleteWithConfirm(new File(SSTable.compactedFilename(dataFilename))); } catch (IOException e) @@ -154,11 +155,21 @@ public abstract class SSTable return Descriptor.fromFilename(dataFile).filenameFor(COMPONENT_FILTER); } + protected static String statisticsFilename(String dataFile) + { + return Descriptor.fromFilename(dataFile).filenameFor(COMPONENT_STATS); + } + public String filterFilename() { return desc.filenameFor(COMPONENT_FILTER); } + public String statisticsFilename() + { + return desc.filenameFor(COMPONENT_STATS); + } + public String getFilename() { return desc.filenameFor(COMPONENT_DATA); Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java?rev=992524&r1=992523&r2=992524&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java Sat Sep 4 03:31:36 2010 @@ -30,7 +30,6 @@ import java.util.TimerTask; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.StatisticsTable; import org.apache.cassandra.io.DeletionService; import org.apache.cassandra.io.util.FileUtils; @@ -95,8 +94,8 @@ public class SSTableDeletingReference ex { FileUtils.deleteWithConfirm(new File(SSTable.indexFilename(path))); FileUtils.deleteWithConfirm(new File(SSTable.filterFilename(path))); + FileUtils.deleteWithConfirm(new File(SSTable.statisticsFilename(path))); FileUtils.deleteWithConfirm(new File(SSTable.compactedFilename(path))); - StatisticsTable.deleteSSTableStatistics(datafile.getAbsolutePath()); } catch (IOException e) { Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=992524&r1=992523&r2=992524&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java Sat Sep 4 03:31:36 2010 @@ -143,18 +143,14 @@ public class SSTableReader extends SSTab private void loadStatistics(Descriptor desc) throws IOException { - // skip loading stats for the system table, or we will infinitely recurse - if (desc.ksname.equals(Table.SYSTEM_TABLE)) + if (!new File(desc.filenameFor(SSTable.COMPONENT_STATS)).exists()) return; if (logger.isDebugEnabled()) logger.debug("Load statistics for " + desc); - long[] rowsizes = StatisticsTable.getSSTableRowSizeStatistics(desc.filenameFor(SSTable.COMPONENT_DATA)); - long[] colcounts = StatisticsTable.getSSTableColumnCountStatistics(desc.filenameFor(SSTable.COMPONENT_DATA)); - if (rowsizes.length > 0) - { - estimatedRowSize = new EstimatedHistogram(rowsizes); - estimatedColumnCount = new EstimatedHistogram(colcounts); - } + DataInputStream dis = new DataInputStream(new FileInputStream(desc.filenameFor(SSTable.COMPONENT_STATS))); + estimatedRowSize = EstimatedHistogram.serializer.deserialize(dis); + estimatedColumnCount = EstimatedHistogram.serializer.deserialize(dis); + dis.close(); } public static SSTableReader open(Descriptor desc) throws IOException Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=992524&r1=992523&r2=992524&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java Sat Sep 4 03:31:36 2010 @@ -37,7 +37,7 @@ import org.apache.cassandra.io.util.Segm import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.BloomFilter; import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.WrappedRunnable; +import org.apache.cassandra.utils.EstimatedHistogram; public class SSTableWriter extends SSTable { @@ -140,17 +140,12 @@ public class SSTableWriter extends SSTab // main data dataFile.close(); // calls force + // write sstable statistics + writeStatistics(desc); + // remove the 'tmp' marker from all components final Descriptor newdesc = rename(desc); - Runnable runnable = new WrappedRunnable() - { - protected void runMayThrow() throws IOException - { - StatisticsTable.persistSSTableStatistics(newdesc, estimatedRowSize, estimatedColumnCount); - } - }; - ColumnFamilyStore.submitPostFlush(runnable); // finalize in-memory state for the reader SegmentedFile ifile = iwriter.builder.complete(newdesc.filenameFor(SSTable.COMPONENT_INDEX)); @@ -161,6 +156,14 @@ public class SSTableWriter extends SSTab return sstable; } + private void writeStatistics(Descriptor desc) throws IOException + { + DataOutputStream dos = new DataOutputStream(new FileOutputStream(desc.filenameFor(SSTable.COMPONENT_STATS))); + EstimatedHistogram.serializer.serialize(estimatedRowSize, dos); + EstimatedHistogram.serializer.serialize(estimatedColumnCount, dos); + dos.close(); + } + static Descriptor rename(Descriptor tmpdesc) { Descriptor newdesc = tmpdesc.asTemporary(false); Modified: cassandra/trunk/src/java/org/apache/cassandra/utils/EstimatedHistogram.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/utils/EstimatedHistogram.java?rev=992524&r1=992523&r2=992524&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/utils/EstimatedHistogram.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/utils/EstimatedHistogram.java Sat Sep 4 03:31:36 2010 @@ -18,8 +18,13 @@ */ package org.apache.cassandra.utils; +import java.io.IOException; import java.util.concurrent.atomic.AtomicLongArray; import java.util.Arrays; +import java.io.DataInputStream; +import java.io.DataOutputStream; + +import org.apache.cassandra.io.ICompactSerializer; public class EstimatedHistogram { @@ -38,6 +43,8 @@ public class EstimatedHistogram final AtomicLongArray buckets; + public static EstimatedHistogramSerializer serializer = new EstimatedHistogramSerializer(); + public EstimatedHistogram() { this(90); @@ -55,6 +62,14 @@ public class EstimatedHistogram buckets = new AtomicLongArray(bucketData); } + public EstimatedHistogram(long[] offsets, long[] bucketData) + { + assert bucketData.length == offsets.length +1; + bucketOffsets = offsets; + buckets = new AtomicLongArray(bucketData); + numBuckets = bucketData.length; + } + private void makeOffsets(int size) { bucketOffsets = new long[size]; @@ -144,4 +159,32 @@ public class EstimatedHistogram } return median; } + + public static class EstimatedHistogramSerializer implements ICompactSerializer + { + public void serialize(EstimatedHistogram eh, DataOutputStream dos) throws IOException + { + long[] offsets = eh.getBucketOffsets(); + long[] buckets = eh.get(false); + dos.writeInt(buckets.length); + for (int i = 0; i < buckets.length; i++) + { + dos.writeLong(offsets[i == 0 ? 0 : i - 1]); + dos.writeLong(buckets[i]); + } + } + + public EstimatedHistogram deserialize(DataInputStream dis) throws IOException + { + int size = dis.readInt(); + long[] offsets = new long[size - 1]; + long[] buckets = new long[size]; + + for (int i = 0; i < size; i++) { + offsets[i == 0 ? 0 : i - 1] = dis.readLong(); + buckets[i] = dis.readLong(); + } + return new EstimatedHistogram(offsets, buckets); + } + } } Modified: cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java?rev=992524&r1=992523&r2=992524&view=diff ============================================================================== --- cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java (original) +++ cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java Sat Sep 4 03:31:36 2010 @@ -51,7 +51,6 @@ public class DefsTest extends CleanupHel assert CFMetaData.HintsCf.cfId == 1; assert CFMetaData.MigrationsCf.cfId == 2; assert CFMetaData.SchemaCf.cfId == 3; - assert CFMetaData.StatisticsCf.cfId == 4; } @Test Modified: cassandra/trunk/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java?rev=992524&r1=992523&r2=992524&view=diff ============================================================================== --- cassandra/trunk/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java (original) +++ cassandra/trunk/test/unit/org/apache/cassandra/db/RecoveryManager2Test.java Sat Sep 4 03:31:36 2010 @@ -48,11 +48,6 @@ public class RecoveryManager2Test extend ColumnFamilyStore cfs = table1.getColumnFamilyStore("Standard1"); cfs.forceBlockingFlush(); - // forceBlockingFlush above adds persistent stats to the current commit log segment - // it ends up in the same segment as key99 meaning that segment still has unwritten data - // thus the commit log replays it when recover is called below - Table.open(Table.SYSTEM_TABLE).getColumnFamilyStore(StatisticsTable.STATISTICS_CF).forceBlockingFlush(); - // remove all SSTable/MemTables cfs.clearUnsafe();