cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marc...@apache.org
Subject [1/2] git commit: Don't keep ancestor information in memory since it is only used on startup.
Date Mon, 08 Jul 2013 10:08:20 GMT
Updated Branches:
  refs/heads/trunk 1a2f35731 -> 07ca445e5


Don't keep ancestor information in memory since it is only used on startup.

patch by marcuse, reviewed by jbellis for CASSANDRA-5342


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/83b75754
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/83b75754
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/83b75754

Branch: refs/heads/trunk
Commit: 83b75754ff143d4d77b01ef76a813da47779c6f4
Parents: cc9ac82
Author: Marcus Eriksson <marcuse@spotify.com>
Authored: Mon Jul 8 09:55:34 2013 +0200
Committer: Marcus Eriksson <marcuse@spotify.com>
Committed: Mon Jul 8 09:57:11 2013 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../cassandra/io/sstable/SSTableMetadata.java   | 39 ++++++++++++++------
 .../cassandra/io/sstable/SSTableReader.java     | 12 +++++-
 .../cassandra/io/sstable/SSTableWriter.java     |  6 +--
 .../cassandra/tools/SSTableMetadataViewer.java  |  2 +-
 .../sstable/SSTableMetadataSerializerTest.java  | 13 +++++--
 6 files changed, 52 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/83b75754/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index fa4df36..b08f967 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@
    (CASSANDRA-5692)
  * Fix skipping range tombstones with reverse queries (CASSANDRA-5712)
  * Expire entries out of ThriftSessionManager (CASSANRDA-5719)
+ * Don't keep ancestor information in memory (CASSANDRA-5342)
 
 1.2.6
  * Fix tracing when operation completes before all responses arrive 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/83b75754/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java b/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
index 865c2ea..de51ea9 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.io.sstable;
 import java.io.*;
 import java.util.*;
 
+import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.StreamingHistogram;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -55,7 +56,6 @@ public class SSTableMetadata
     public final long maxTimestamp;
     public final double compressionRatio;
     public final String partitioner;
-    public final Set<Integer> ancestors;
     public final StreamingHistogram estimatedTombstoneDropTime;
 
     private SSTableMetadata()
@@ -67,7 +67,6 @@ public class SSTableMetadata
              Long.MAX_VALUE,
              NO_COMPRESSION_RATIO,
              null,
-             Collections.<Integer>emptySet(),
              defaultTombstoneDropTimeHistogram());
     }
 
@@ -78,7 +77,6 @@ public class SSTableMetadata
                             long maxTimestamp,
                             double cr,
                             String partitioner,
-                            Set<Integer> ancestors,
                             StreamingHistogram estimatedTombstoneDropTime)
     {
         this.estimatedRowSize = rowSizes;
@@ -88,7 +86,6 @@ public class SSTableMetadata
         this.maxTimestamp = maxTimestamp;
         this.compressionRatio = cr;
         this.partitioner = partitioner;
-        this.ancestors = ancestors;
         this.estimatedTombstoneDropTime = estimatedTombstoneDropTime;
     }
 
@@ -193,7 +190,6 @@ public class SSTableMetadata
                                        maxTimestamp,
                                        compressionRatio,
                                        partitioner,
-                                       ancestors,
                                        estimatedTombstoneDropTime);
         }
 
@@ -242,7 +238,7 @@ public class SSTableMetadata
     {
         private static final Logger logger = LoggerFactory.getLogger(SSTableMetadataSerializer.class);
 
-        public void serialize(SSTableMetadata sstableStats, DataOutput dos) throws IOException
+        public void serialize(SSTableMetadata sstableStats, Set<Integer> ancestors,
DataOutput dos) throws IOException
         {
             assert sstableStats.partitioner != null;
 
@@ -253,20 +249,31 @@ public class SSTableMetadata
             dos.writeLong(sstableStats.maxTimestamp);
             dos.writeDouble(sstableStats.compressionRatio);
             dos.writeUTF(sstableStats.partitioner);
-            dos.writeInt(sstableStats.ancestors.size());
-            for (Integer g : sstableStats.ancestors)
+            dos.writeInt(ancestors.size());
+            for (Integer g : ancestors)
                 dos.writeInt(g);
             StreamingHistogram.serializer.serialize(sstableStats.estimatedTombstoneDropTime,
dos);
         }
 
-        public SSTableMetadata deserialize(Descriptor descriptor) throws IOException
+        /**
+         * deserializes the metadata
+         *
+         * returns a pair containing the part of the metadata meant to be kept-in memory
and the part
+         * that should not.
+         *
+         * @param descriptor the descriptor
+         * @return a pair containing data that needs to be in memory and data that is potentially
big and is not needed
+         *         in memory
+         * @throws IOException
+         */
+        public Pair<SSTableMetadata, Set<Integer>> deserialize(Descriptor descriptor)
throws IOException
         {
             logger.debug("Load metadata for {}", descriptor);
             File statsFile = new File(descriptor.filenameFor(SSTable.COMPONENT_STATS));
             if (!statsFile.exists())
             {
                 logger.debug("No sstable stats for {}", descriptor);
-                return new SSTableMetadata();
+                return Pair.create(new SSTableMetadata(), Collections.<Integer>emptySet());
             }
 
             DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(statsFile)));
@@ -280,7 +287,7 @@ public class SSTableMetadata
             }
         }
 
-        public SSTableMetadata deserialize(DataInputStream dis, Descriptor desc) throws IOException
+        public Pair<SSTableMetadata, Set<Integer>> deserialize(DataInputStream
dis, Descriptor desc) throws IOException
         {
             EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(dis);
             EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(dis);
@@ -308,7 +315,15 @@ public class SSTableMetadata
             StreamingHistogram tombstoneHistogram = desc.version.tracksTombstones
                                                    ? StreamingHistogram.serializer.deserialize(dis)
                                                    : defaultTombstoneDropTimeHistogram();
-            return new SSTableMetadata(rowSizes, columnCounts, replayPosition, minTimestamp,
maxTimestamp, compressionRatio, partitioner, ancestors, tombstoneHistogram);
+            return Pair.create(new SSTableMetadata(rowSizes,
+                                                   columnCounts,
+                                                   replayPosition,
+                                                   minTimestamp,
+                                                   maxTimestamp,
+                                                   compressionRatio,
+                                                   partitioner,
+                                                   tombstoneHistogram),
+                                                   ancestors);
         }
     }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/83b75754/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
index ea9c451..ab58c11 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
@@ -168,7 +168,7 @@ public class SSTableReader extends SSTable
         long start = System.currentTimeMillis();
         logger.info("Opening {} ({} bytes)", descriptor, new File(descriptor.filenameFor(COMPONENT_DATA)).length());
 
-        SSTableMetadata sstableMetadata = SSTableMetadata.serializer.deserialize(descriptor);
+        SSTableMetadata sstableMetadata = SSTableMetadata.serializer.deserialize(descriptor).left;
 
         // Check if sstable is created using same partitioner.
         // Partitioner can be null, which indicates older version of sstable or no stats
available.
@@ -1116,7 +1116,15 @@ public class SSTableReader extends SSTable
 
     public Set<Integer> getAncestors()
     {
-        return sstableMetadata.ancestors;
+        try
+        {
+            return SSTableMetadata.serializer.deserialize(descriptor).right;
+        }
+        catch (IOException e)
+        {
+            SSTableReader.logOpenException(descriptor, e);
+            return Collections.emptySet();
+        }
     }
 
     public RandomAccessReader openDataReader(RateLimiter limiter)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/83b75754/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
index 51aeaa6..54e154c 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
@@ -328,7 +328,7 @@ public class SSTableWriter extends SSTable
         dataFile.close();
         // write sstable statistics
         SSTableMetadata sstableMetadata = sstableMetadataCollector.finalizeMetadata(partitioner.getClass().getCanonicalName());
-        writeMetadata(descriptor, sstableMetadata);
+        writeMetadata(descriptor, sstableMetadata, sstableMetadataCollector.ancestors);
         maybeWriteDigest();
 
         // save the table of components
@@ -381,12 +381,12 @@ public class SSTableWriter extends SSTable
         out.close();
     }
 
-    private static void writeMetadata(Descriptor desc, SSTableMetadata sstableMetadata)
+    private static void writeMetadata(Descriptor desc, SSTableMetadata sstableMetadata, Set<Integer>
ancestors)
     {
         SequentialWriter out = SequentialWriter.open(new File(desc.filenameFor(SSTable.COMPONENT_STATS)),
true);
         try
         {
-            SSTableMetadata.serializer.serialize(sstableMetadata, out.stream);
+            SSTableMetadata.serializer.serialize(sstableMetadata, ancestors, out.stream);
         }
         catch (IOException e)
         {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/83b75754/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
index c1f0332..55f2769 100644
--- a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
+++ b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
@@ -44,7 +44,7 @@ public class SSTableMetadataViewer
         for (String fname : args)
         {
             Descriptor descriptor = Descriptor.fromFilename(fname);
-            SSTableMetadata metadata = SSTableMetadata.serializer.deserialize(descriptor);
+            SSTableMetadata metadata = SSTableMetadata.serializer.deserialize(descriptor).left;
 
             out.printf("SSTable: %s%n", descriptor);
             out.printf("Partitioner: %s%n", metadata.partitioner);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/83b75754/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java
b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java
index ab96f89..3e8ef42 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java
@@ -24,12 +24,16 @@ import java.io.DataOutputStream;
 import java.io.DataInputStream;
 import java.io.File;
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
 
 import org.junit.Test;
 
 import org.apache.cassandra.db.commitlog.ReplayPosition;
 import org.apache.cassandra.dht.RandomPartitioner;
 import org.apache.cassandra.utils.EstimatedHistogram;
+import org.apache.cassandra.utils.Pair;
 
 public class SSTableMetadataSerializerTest
 {
@@ -56,13 +60,15 @@ public class SSTableMetadataSerializerTest
 
         ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
         DataOutputStream dos = new DataOutputStream(byteOutput);
-
-        SSTableMetadata.serializer.serialize(originalMetadata, dos);
+        Set<Integer> ancestors = new HashSet<Integer>();
+        ancestors.addAll(Arrays.asList(1,2,3,4));
+        SSTableMetadata.serializer.serialize(originalMetadata, ancestors, dos);
 
         ByteArrayInputStream byteInput = new ByteArrayInputStream(byteOutput.toByteArray());
         DataInputStream dis = new DataInputStream(byteInput);
         Descriptor desc = new Descriptor(Descriptor.Version.CURRENT, new File("."), "", "",
0, false);
-        SSTableMetadata stats = SSTableMetadata.serializer.deserialize(dis, desc);
+        Pair<SSTableMetadata, Set<Integer>> statsPair = SSTableMetadata.serializer.deserialize(dis,
desc);
+        SSTableMetadata stats = statsPair.left;
 
         assert stats.estimatedRowSize.equals(originalMetadata.estimatedRowSize);
         assert stats.estimatedRowSize.equals(rowSizes);
@@ -75,5 +81,6 @@ public class SSTableMetadataSerializerTest
         assert stats.minTimestamp == originalMetadata.minTimestamp;
         assert stats.maxTimestamp == originalMetadata.maxTimestamp;
         assert RandomPartitioner.class.getCanonicalName().equals(stats.partitioner);
+        assert ancestors.equals(statsPair.right);
     }
 }


Mime
View raw message