cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marc...@apache.org
Subject [2/2] git commit: Merge branch 'cassandra-1.2' into trunk
Date Mon, 08 Jul 2013 10:08:21 GMT
Merge branch 'cassandra-1.2' into trunk

Conflicts:
	src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
	src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
	test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/07ca445e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/07ca445e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/07ca445e

Branch: refs/heads/trunk
Commit: 07ca445e5b1a2abce44837c926321dc3bea2749e
Parents: 1a2f357 83b7575
Author: Marcus Eriksson <marcuse@spotify.com>
Authored: Mon Jul 8 11:40:27 2013 +0200
Committer: Marcus Eriksson <marcuse@spotify.com>
Committed: Mon Jul 8 11:51:07 2013 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  |  7 ++-
 .../db/compaction/LegacyLeveledManifest.java    |  3 +-
 .../db/compaction/LeveledManifest.java          |  8 ++--
 .../cassandra/io/sstable/SSTableMetadata.java   | 45 +++++++++++---------
 .../cassandra/io/sstable/SSTableReader.java     | 14 ++++--
 .../cassandra/io/sstable/SSTableWriter.java     |  6 +--
 .../cassandra/tools/SSTableLevelResetter.java   |  5 ++-
 .../cassandra/tools/SSTableMetadataViewer.java  |  2 +-
 .../compaction/LegacyLeveledManifestTest.java   | 15 ++++---
 .../LeveledCompactionStrategyTest.java          |  3 +-
 .../sstable/SSTableMetadataSerializerTest.java  | 13 +++++-
 12 files changed, 75 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/07ca445e/CHANGES.txt
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/07ca445e/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 643f405,429859e..4b88a42
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -458,77 -429,11 +458,76 @@@ public class ColumnFamilyStore implemen
          }
  
          // also clean out any index leftovers.
 -        CFMetaData cfm = Schema.instance.getCFMetaData(table, columnFamily);
 +        CFMetaData cfm = Schema.instance.getCFMetaData(keyspaceName, columnFamily);
          if (cfm != null) // secondary indexes aren't stored in DD.
          {
 -            for (ColumnDefinition def : cfm.getColumn_metadata().values())
 -                scrubDataDirectories(table, cfm.indexColumnFamilyName(def));
 +            for (ColumnDefinition def : cfm.allColumns())
 +                scrubDataDirectories(keyspaceName, cfm.indexColumnFamilyName(def));
 +        }
 +    }
 +
 +    /**
 +     * Replacing compacted sstables is atomic as far as observers of DataTracker are concerned, but not on the
 +     * filesystem: first the new sstables are renamed to "live" status (i.e., the tmp marker is removed), then
 +     * their ancestors are removed.
 +     *
 +     * If an unclean shutdown happens at the right time, we can thus end up with both the new ones and their
 +     * ancestors "live" in the system.  This is harmless for normal data, but for counters it can cause overcounts.
 +     *
 +     * To prevent this, we record sstables being compacted in the system keyspace.  If we find unfinished
 +     * compactions, we remove the new ones (since those may be incomplete -- under LCS, we may create multiple
 +     * sstables from any given ancestor).
 +     */
 +    public static void removeUnfinishedCompactionLeftovers(String keyspace, String columnfamily, Set<Integer> unfinishedGenerations)
 +    {
 +        Directories directories = Directories.create(keyspace, columnfamily);
 +
 +        // sanity-check unfinishedGenerations
 +        Set<Integer> allGenerations = new HashSet<Integer>();
 +        for (Descriptor desc : directories.sstableLister().list().keySet())
 +            allGenerations.add(desc.generation);
 +        if (!allGenerations.containsAll(unfinishedGenerations))
 +        {
 +            throw new IllegalStateException("Unfinished compactions reference missing sstables."
 +                                            + " This should never happen since compactions are marked finished before we start removing the old sstables.");
 +        }
 +
 +        // remove new sstables from compactions that didn't complete, and compute
 +        // set of ancestors that shouldn't exist anymore
 +        Set<Integer> completedAncestors = new HashSet<Integer>();
 +        for (Map.Entry<Descriptor, Set<Component>> sstableFiles : directories.sstableLister().list().entrySet())
 +        {
 +            Descriptor desc = sstableFiles.getKey();
 +            Set<Component> components = sstableFiles.getValue();
 +
-             SSTableMetadata meta;
++            Set<Integer> ancestors;
 +            try
 +            {
-                 meta = SSTableMetadata.serializer.deserialize(desc);
++                ancestors = SSTableMetadata.serializer.deserialize(desc).right;
 +            }
 +            catch (IOException e)
 +            {
 +                throw new FSReadError(e, desc.filenameFor(Component.STATS));
 +            }
 +
-             Set<Integer> ancestors = meta.ancestors;
 +            if (!ancestors.isEmpty() && unfinishedGenerations.containsAll(ancestors))
 +            {
 +                SSTable.delete(desc, components);
 +            }
 +            else
 +            {
 +                completedAncestors.addAll(ancestors);
 +            }
 +        }
 +
 +        // remove old sstables from compactions that did complete
 +        for (Map.Entry<Descriptor, Set<Component>> sstableFiles : directories.sstableLister().list().entrySet())
 +        {
 +            Descriptor desc = sstableFiles.getKey();
 +            Set<Component> components = sstableFiles.getValue();
 +
 +            if (completedAncestors.contains(desc.generation))
 +                SSTable.delete(desc, components);
          }
      }
  
@@@ -589,21 -494,6 +588,21 @@@
                                                           Descriptor.Version.CURRENT,
                                                           descriptor));
  
 +            // force foreign sstables to level 0
 +            try
 +            {
 +                if (new File(descriptor.filenameFor(Component.STATS)).exists())
 +                {
-                     SSTableMetadata oldMetadata = SSTableMetadata.serializer.deserialize(descriptor);
++                    Pair<SSTableMetadata, Set<Integer>> oldMetadata = SSTableMetadata.serializer.deserialize(descriptor);
 +                    LeveledManifest.mutateLevel(oldMetadata, descriptor, descriptor.filenameFor(Component.STATS), 0);
 +                }
 +            }
 +            catch (IOException e)
 +            {
 +                SSTableReader.logOpenException(entry.getKey(), e);
 +                continue;
 +            }
 +
              Descriptor newDescriptor = new Descriptor(descriptor.version,
                                                        descriptor.directory,
                                                        descriptor.ksname,

http://git-wip-us.apache.org/repos/asf/cassandra/blob/07ca445e/src/java/org/apache/cassandra/db/compaction/LegacyLeveledManifest.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/LegacyLeveledManifest.java
index 8403e75,0000000..645c2be
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/compaction/LegacyLeveledManifest.java
+++ b/src/java/org/apache/cassandra/db/compaction/LegacyLeveledManifest.java
@@@ -1,140 -1,0 +1,141 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.cassandra.db.compaction;
 +
 +import java.io.File;
 +import java.io.IOException;
 +import java.util.HashMap;
 +import java.util.Map;
 +import java.util.Set;
 +
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +import org.apache.cassandra.db.Directories;
 +import org.apache.cassandra.io.sstable.Component;
 +import org.apache.cassandra.io.sstable.Descriptor;
 +import org.apache.cassandra.io.sstable.SSTableMetadata;
 +import org.apache.cassandra.io.util.FileUtils;
++import org.apache.cassandra.utils.Pair;
 +import org.codehaus.jackson.JsonNode;
 +import org.codehaus.jackson.map.ObjectMapper;
 +
 +/**
 + * This class was added to be able to migrate pre-CASSANDRA-4782 leveled manifests into the sstable metadata
 + *
 + * @deprecated since it can be removed in a future revision.
 + */
 +@Deprecated
 +public class LegacyLeveledManifest
 +{
 +    private static final Logger logger = LoggerFactory.getLogger(LegacyLeveledManifest.class);
 +
 +    private Map<Integer, Integer> sstableLevels;
 +
 +    private LegacyLeveledManifest(File path) throws IOException
 +    {
 +        sstableLevels = new HashMap<Integer, Integer>();
 +        ObjectMapper m = new ObjectMapper();
 +        JsonNode rootNode = m.readValue(path, JsonNode.class);
 +        JsonNode generations = rootNode.get("generations");
 +        assert generations.isArray();
 +        for (JsonNode generation : generations)
 +        {
 +            int level = generation.get("generation").getIntValue();
 +            JsonNode generationValues = generation.get("members");
 +            for (JsonNode generationValue : generationValues)
 +            {
 +                sstableLevels.put(generationValue.getIntValue(), level);
 +            }
 +        }
 +    }
 +
 +    private int levelOf(int sstableGeneration)
 +    {
 +        return sstableLevels.containsKey(sstableGeneration) ? sstableLevels.get(sstableGeneration) : 0;
 +    }
 +
 +    /**
 +     * We need to migrate if there is a legacy leveledmanifest json-file
 +     * <p/>
 +     * If there is no jsonfile, we can just start as normally, sstable level will be at 0 for all sstables.
 +     *
 +     * @param keyspace
 +     * @param columnFamily
 +     * @return
 +     */
 +    public static boolean manifestNeedsMigration(String keyspace, String columnFamily)
 +    {
 +        return Directories.create(keyspace, columnFamily).tryGetLeveledManifest() != null;
 +    }
 +
 +    public static void migrateManifests(String keyspace, String columnFamily) throws IOException
 +    {
 +        logger.info("Migrating manifest for {}/{}", keyspace, columnFamily);
 +
 +        snapshotWithoutCFS(keyspace, columnFamily);
 +        Directories directories = Directories.create(keyspace, columnFamily);
 +        File manifestFile = directories.tryGetLeveledManifest();
 +        if (manifestFile == null)
 +            return;
 +
 +        LegacyLeveledManifest legacyManifest = new LegacyLeveledManifest(manifestFile);
 +        for (Map.Entry<Descriptor, Set<Component>> entry : directories.sstableLister().includeBackups(false).skipTemporary(true).list().entrySet())
 +        {
 +            Descriptor d = entry.getKey();
-             SSTableMetadata oldMetadata = SSTableMetadata.serializer.deserialize(d, false);
++            Pair<SSTableMetadata, Set<Integer>> oldMetadata = SSTableMetadata.serializer.deserialize(d, false);
 +            String metadataFilename = d.filenameFor(Component.STATS);
 +            LeveledManifest.mutateLevel(oldMetadata, d, metadataFilename, legacyManifest.levelOf(d.generation));
 +        }
 +        FileUtils.deleteWithConfirm(manifestFile);
 +    }
 +
 +    /**
 +     * Snapshot a CF without having to load the sstables in that directory
 +     *
 +     * @param keyspace
 +     * @param columnFamily
 +     * @throws IOException
 +     */
 +    public static void snapshotWithoutCFS(String keyspace, String columnFamily) throws IOException
 +    {
 +        Directories directories = Directories.create(keyspace, columnFamily);
 +        String snapshotName = "pre-sstablemetamigration";
 +        logger.info("Snapshotting {}, {} to {}", keyspace, columnFamily, snapshotName);
 +
 +        for (Map.Entry<Descriptor, Set<Component>> entry : directories.sstableLister().includeBackups(false).skipTemporary(true).list().entrySet())
 +        {
 +            Descriptor descriptor = entry.getKey();
 +            File snapshotDirectoryPath = Directories.getSnapshotDirectory(descriptor, snapshotName);
 +            for (Component component : entry.getValue())
 +            {
 +                File sourceFile = new File(descriptor.filenameFor(component));
 +                File targetLink = new File(snapshotDirectoryPath, sourceFile.getName());
 +                FileUtils.createHardLink(sourceFile, targetLink);
 +            }
 +        }
 +
 +        File manifestFile = directories.tryGetLeveledManifest();
 +        if (manifestFile != null)
 +        {
 +            File snapshotDirectory = new File(new File(manifestFile.getParentFile(), Directories.SNAPSHOT_SUBDIR), snapshotName);
 +            File target = new File(snapshotDirectory, manifestFile.getName());
 +            FileUtils.createHardLink(manifestFile, target);
 +        }
 +    }
 +}

http://git-wip-us.apache.org/repos/asf/cassandra/blob/07ca445e/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
index c0404e3,f62c796..7a0a927
--- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
@@@ -184,17 -262,23 +184,17 @@@ public class LeveledManifes
      private synchronized void sendBackToL0(SSTableReader sstable)
      {
          remove(sstable);
 -        add(sstable, 0);
 +        String metaDataFile = sstable.descriptor.filenameFor(Component.STATS);
 +        try
 +        {
-             mutateLevel(sstable.getSSTableMetadata(), sstable.descriptor, metaDataFile, 0);
++            mutateLevel(Pair.create(sstable.getSSTableMetadata(), sstable.getAncestors()), sstable.descriptor, metaDataFile, 0);
 +            sstable.reloadSSTableMetadata();
 +            add(sstable);
 +        }
 +        catch (IOException e)
 +        {
 +            throw new RuntimeException("Could not reload sstable meta data", e);
 +        }
      }
  
      private String toString(Iterable<SSTableReader> sstables)
@@@ -541,58 -668,7 +541,58 @@@
          }
  
          logger.debug("Estimating {} compactions to do for {}.{}",
 -                     new Object[] {Arrays.toString(estimated), cfs.table.name, cfs.columnFamily});
 +                     Arrays.toString(estimated), cfs.keyspace.getName(), cfs.name);
          return Ints.checkedCast(tasks);
      }
 +
 +    public int getNextLevel(Collection<SSTableReader> sstables)
 +    {
 +        int maximumLevel = Integer.MIN_VALUE;
 +        int minimumLevel = Integer.MAX_VALUE;
 +        for (SSTableReader sstable : sstables)
 +        {
 +            maximumLevel = Math.max(sstable.getSSTableLevel(), maximumLevel);
 +            minimumLevel = Math.min(sstable.getSSTableLevel(), minimumLevel);
 +        }
 +
 +        int newLevel;
 +        if (minimumLevel == 0 && minimumLevel == maximumLevel && SSTable.getTotalBytes(sstables) < maxSSTableSizeInBytes)
 +        {
 +            newLevel = 0;
 +        }
 +        else
 +        {
 +            newLevel = minimumLevel == maximumLevel ? maximumLevel + 1 : maximumLevel;
 +            newLevel = skipLevels(newLevel, sstables);
 +            assert newLevel > 0;
 +        }
 +        return newLevel;
 +
 +    }
 +
 +    /**
 +     * Scary method mutating existing sstable component
 +     *
 +     * Tries to do it safely by moving the new file on top of the old one
 +     *
 +     * Caller needs to reload the sstable metadata (sstableReader.reloadSSTableMetadata())
 +     *
 +     * @see org.apache.cassandra.io.sstable.SSTableReader#reloadSSTableMetadata()
 +     *
 +     * @param oldMetadata
 +     * @param descriptor
 +     * @param filename
 +     * @param level
 +     * @throws IOException
 +     */
-     public static synchronized void mutateLevel(SSTableMetadata oldMetadata, Descriptor descriptor, String filename, int level) throws IOException
++    public static synchronized void mutateLevel(Pair<SSTableMetadata, Set<Integer>> oldMetadata, Descriptor descriptor, String filename, int level) throws IOException
 +    {
 +        logger.debug("Mutating {} to level {}", descriptor.filenameFor(Component.STATS), level);
-         SSTableMetadata metadata = SSTableMetadata.copyWithNewSSTableLevel(oldMetadata, level);
++        SSTableMetadata metadata = SSTableMetadata.copyWithNewSSTableLevel(oldMetadata.left, level);
 +        DataOutputStream out = new DataOutputStream(new FileOutputStream(filename + "-tmp"));
-         SSTableMetadata.serializer.legacySerialize(metadata, descriptor, out);
++        SSTableMetadata.serializer.legacySerialize(metadata, oldMetadata.right, descriptor, out);
 +        out.flush();
 +        out.close();
 +        FileUtils.renameWithConfirm(filename + "-tmp", filename);
 +    }
  }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/07ca445e/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
index ff068e8,de51ea9..140e08b
--- a/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
@@@ -18,11 -18,9 +18,12 @@@
  package org.apache.cassandra.io.sstable;
  
  import java.io.*;
 +import java.nio.ByteBuffer;
  import java.util.*;
  
 +import org.apache.cassandra.db.marshal.AbstractType;
 +import org.apache.cassandra.utils.ByteBufferUtil;
+ import org.apache.cassandra.utils.Pair;
  import org.apache.cassandra.utils.StreamingHistogram;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
@@@ -59,15 -54,9 +60,14 @@@ public class SSTableMetadat
      public final ReplayPosition replayPosition;
      public final long minTimestamp;
      public final long maxTimestamp;
 +    public final int maxLocalDeletionTime;
 +    public final double bloomFilterFPChance;
      public final double compressionRatio;
      public final String partitioner;
-     public final Set<Integer> ancestors;
      public final StreamingHistogram estimatedTombstoneDropTime;
 +    public final int sstableLevel;
 +    public final List<ByteBuffer> maxColumnNames;
 +    public final List<ByteBuffer> minColumnNames;
  
      private SSTableMetadata()
      {
@@@ -76,15 -65,9 +76,14 @@@
               ReplayPosition.NONE,
               Long.MIN_VALUE,
               Long.MAX_VALUE,
 +             Integer.MAX_VALUE,
 +             NO_BLOOM_FLITER_FP_CHANCE,
               NO_COMPRESSION_RATIO,
               null,
-              Collections.<Integer>emptySet(),
 -             defaultTombstoneDropTimeHistogram());
 +             defaultTombstoneDropTimeHistogram(),
 +             0,
 +             Collections.<ByteBuffer>emptyList(),
 +             Collections.<ByteBuffer>emptyList());
      }
  
      private SSTableMetadata(EstimatedHistogram rowSizes,
@@@ -92,82 -75,23 +91,79 @@@
                              ReplayPosition replayPosition,
                              long minTimestamp,
                              long maxTimestamp,
 -                            double cr,
 +                            int maxLocalDeletionTime,
 +                            double bloomFilterFPChance,
 +                            double compressionRatio,
                              String partitioner,
-                             Set<Integer> ancestors,
 -                            StreamingHistogram estimatedTombstoneDropTime)
 +                            StreamingHistogram estimatedTombstoneDropTime,
 +                            int sstableLevel,
 +                            List<ByteBuffer> minColumnNames,
 +                            List<ByteBuffer> maxColumnNames)
      {
          this.estimatedRowSize = rowSizes;
          this.estimatedColumnCount = columnCounts;
          this.replayPosition = replayPosition;
          this.minTimestamp = minTimestamp;
          this.maxTimestamp = maxTimestamp;
 -        this.compressionRatio = cr;
 +        this.maxLocalDeletionTime = maxLocalDeletionTime;
 +        this.bloomFilterFPChance = bloomFilterFPChance;
 +        this.compressionRatio = compressionRatio;
          this.partitioner = partitioner;
-         this.ancestors = ancestors;
          this.estimatedTombstoneDropTime = estimatedTombstoneDropTime;
 +        this.sstableLevel = sstableLevel;
 +        this.minColumnNames = minColumnNames;
 +        this.maxColumnNames = maxColumnNames;
      }
  
 -    public static Collector createCollector()
 +    public static Collector createCollector(AbstractType<?> columnNameComparator)
      {
 -        return new Collector();
 +        return new Collector(columnNameComparator);
 +    }
 +
 +    public static Collector createCollector(Collection<SSTableReader> sstables, AbstractType<?> columnNameComparator, int level)
 +    {
 +        Collector collector = new Collector(columnNameComparator);
 +
 +        collector.replayPosition(ReplayPosition.getReplayPosition(sstables));
 +        collector.sstableLevel(level);
 +        // Get the max timestamp of the precompacted sstables
 +        // and adds generation of live ancestors
 +        for (SSTableReader sstable : sstables)
 +        {
 +            collector.addAncestor(sstable.descriptor.generation);
 +            for (Integer i : sstable.getAncestors())
 +            {
 +                if (new File(sstable.descriptor.withGeneration(i).filenameFor(Component.DATA)).exists())
 +                    collector.addAncestor(i);
 +            }
 +        }
 +
 +        return collector;
 +    }
 +
 +    /**
 +     * Used when updating sstablemetadata files with an sstable level
 +     * @param metadata
 +     * @param sstableLevel
 +     * @return
 +     */
 +    @Deprecated
 +    public static SSTableMetadata copyWithNewSSTableLevel(SSTableMetadata metadata, int sstableLevel)
 +    {
 +        return new SSTableMetadata(metadata.estimatedRowSize,
 +                                   metadata.estimatedColumnCount,
 +                                   metadata.replayPosition,
 +                                   metadata.minTimestamp,
 +                                   metadata.maxTimestamp,
 +                                   metadata.maxLocalDeletionTime,
 +                                   metadata.bloomFilterFPChance,
 +                                   metadata.compressionRatio,
 +                                   metadata.partitioner,
-                                    metadata.ancestors,
 +                                   metadata.estimatedTombstoneDropTime,
 +                                   sstableLevel,
 +                                   metadata.minColumnNames,
 +                                   metadata.maxColumnNames);
 +
      }
  
      static EstimatedHistogram defaultColumnCountHistogram()
@@@ -278,15 -188,9 +274,14 @@@
                                         replayPosition,
                                         minTimestamp,
                                         maxTimestamp,
 +                                       maxLocalDeletionTime,
 +                                       bloomFilterFPChance,
                                         compressionRatio,
                                         partitioner,
-                                        ancestors,
 -                                       estimatedTombstoneDropTime);
 +                                       estimatedTombstoneDropTime,
 +                                       sstableLevel,
 +                                       minColumnNames,
 +                                       maxColumnNames);
          }
  
          public Collector estimatedRowSize(EstimatedHistogram estimatedRowSize)
@@@ -357,157 -238,92 +352,167 @@@
      {
          private static final Logger logger = LoggerFactory.getLogger(SSTableMetadataSerializer.class);
  
-         public void serialize(SSTableMetadata sstableStats, DataOutput out) throws IOException
 -        public void serialize(SSTableMetadata sstableStats, Set<Integer> ancestors, DataOutput dos) throws IOException
++        public void serialize(SSTableMetadata sstableStats, Set<Integer> ancestors, DataOutput out) throws IOException
          {
              assert sstableStats.partitioner != null;
  
 -            EstimatedHistogram.serializer.serialize(sstableStats.estimatedRowSize, dos);
 -            EstimatedHistogram.serializer.serialize(sstableStats.estimatedColumnCount, dos);
 -            ReplayPosition.serializer.serialize(sstableStats.replayPosition, dos);
 -            dos.writeLong(sstableStats.minTimestamp);
 -            dos.writeLong(sstableStats.maxTimestamp);
 -            dos.writeDouble(sstableStats.compressionRatio);
 -            dos.writeUTF(sstableStats.partitioner);
 -            dos.writeInt(ancestors.size());
 +            EstimatedHistogram.serializer.serialize(sstableStats.estimatedRowSize, out);
 +            EstimatedHistogram.serializer.serialize(sstableStats.estimatedColumnCount, out);
 +            ReplayPosition.serializer.serialize(sstableStats.replayPosition, out);
 +            out.writeLong(sstableStats.minTimestamp);
 +            out.writeLong(sstableStats.maxTimestamp);
 +            out.writeInt(sstableStats.maxLocalDeletionTime);
 +            out.writeDouble(sstableStats.bloomFilterFPChance);
 +            out.writeDouble(sstableStats.compressionRatio);
 +            out.writeUTF(sstableStats.partitioner);
-             out.writeInt(sstableStats.ancestors.size());
-             for (Integer g : sstableStats.ancestors)
++            out.writeInt(ancestors.size());
+             for (Integer g : ancestors)
 -                dos.writeInt(g);
 -            StreamingHistogram.serializer.serialize(sstableStats.estimatedTombstoneDropTime, dos);
 +                out.writeInt(g);
 +            StreamingHistogram.serializer.serialize(sstableStats.estimatedTombstoneDropTime, out);
 +            out.writeInt(sstableStats.sstableLevel);
 +            serializeMinMaxColumnNames(sstableStats.minColumnNames, sstableStats.maxColumnNames, out);
 +        }
 +
 +        private void serializeMinMaxColumnNames(List<ByteBuffer> minColNames, List<ByteBuffer> maxColNames, DataOutput out) throws IOException
 +        {
 +            out.writeInt(minColNames.size());
 +            for (ByteBuffer columnName : minColNames)
 +                ByteBufferUtil.writeWithShortLength(columnName, out);
 +            out.writeInt(maxColNames.size());
 +            for (ByteBuffer columnName : maxColNames)
 +                ByteBufferUtil.writeWithShortLength(columnName, out);
 +        }
 +        /**
 +         * Used to serialize to an old version - needed to be able to update sstable level without a full compaction.
 +         *
 +         * @deprecated will be removed when it is assumed that the minimum upgrade-from-version is the version that this
 +         * patch made it into
 +         *
 +         * @param sstableStats
 +         * @param legacyDesc
 +         * @param out
 +         * @throws IOException
 +         */
 +        @Deprecated
-         public void legacySerialize(SSTableMetadata sstableStats, Descriptor legacyDesc, DataOutput out) throws IOException
++        public void legacySerialize(SSTableMetadata sstableStats, Set<Integer> ancestors, Descriptor legacyDesc, DataOutput out) throws IOException
 +        {
 +            EstimatedHistogram.serializer.serialize(sstableStats.estimatedRowSize, out);
 +            EstimatedHistogram.serializer.serialize(sstableStats.estimatedColumnCount, out);
 +            ReplayPosition.serializer.serialize(sstableStats.replayPosition, out);
 +            out.writeLong(sstableStats.minTimestamp);
 +            out.writeLong(sstableStats.maxTimestamp);
 +            if (legacyDesc.version.tracksMaxLocalDeletionTime)
 +                out.writeInt(sstableStats.maxLocalDeletionTime);
 +            if (legacyDesc.version.hasBloomFilterFPChance)
 +                out.writeDouble(sstableStats.bloomFilterFPChance);
 +            out.writeDouble(sstableStats.compressionRatio);
 +            out.writeUTF(sstableStats.partitioner);
-             out.writeInt(sstableStats.ancestors.size());
-             for (Integer g : sstableStats.ancestors)
++            out.writeInt(ancestors.size());
++            for (Integer g : ancestors)
 +                out.writeInt(g);
 +            StreamingHistogram.serializer.serialize(sstableStats.estimatedTombstoneDropTime, out);
 +            out.writeInt(sstableStats.sstableLevel);
 +            if (legacyDesc.version.tracksMaxMinColumnNames)
 +                serializeMinMaxColumnNames(sstableStats.minColumnNames, sstableStats.maxColumnNames, out);
          }
  
-         public SSTableMetadata deserialize(Descriptor descriptor) throws IOException
+         /**
+          * deserializes the metadata
+          *
+          * returns a pair containing the part of the metadata meant to be kept-in memory and the part
+          * that should not.
+          *
+          * @param descriptor the descriptor
+          * @return a pair containing data that needs to be in memory and data that is potentially big and is not needed
+          *         in memory
+          * @throws IOException
+          */
+         public Pair<SSTableMetadata, Set<Integer>> deserialize(Descriptor descriptor) throws IOException
          {
 +            return deserialize(descriptor, true);
 +        }
 +
-         public SSTableMetadata deserialize(Descriptor descriptor, boolean loadSSTableLevel) throws IOException
++        public Pair<SSTableMetadata, Set<Integer>> deserialize(Descriptor descriptor, boolean loadSSTableLevel) throws IOException
 +        {
              logger.debug("Load metadata for {}", descriptor);
              File statsFile = new File(descriptor.filenameFor(SSTable.COMPONENT_STATS));
              if (!statsFile.exists())
              {
                  logger.debug("No sstable stats for {}", descriptor);
-                 return new SSTableMetadata();
+                 return Pair.create(new SSTableMetadata(), Collections.<Integer>emptySet());
              }
  
 -            DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(statsFile)));
 +            DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(statsFile)));
              try
              {
 -                return deserialize(dis, descriptor);
 +                return deserialize(in, descriptor, loadSSTableLevel);
              }
              finally
              {
 -                FileUtils.closeQuietly(dis);
 +                FileUtils.closeQuietly(in);
              }
          }
-         public SSTableMetadata deserialize(DataInputStream in, Descriptor desc) throws IOException
++        public Pair<SSTableMetadata, Set<Integer>> deserialize(DataInputStream in, Descriptor desc) throws IOException
 +        {
 +            return deserialize(in, desc, true);
 +        }
  
-         public SSTableMetadata deserialize(DataInputStream in, Descriptor desc, boolean loadSSTableLevel) throws IOException
 -        public Pair<SSTableMetadata, Set<Integer>> deserialize(DataInputStream dis, Descriptor desc) throws IOException
++        public Pair<SSTableMetadata, Set<Integer>> deserialize(DataInputStream in, Descriptor desc, boolean loadSSTableLevel) throws IOException
          {
 -            EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(dis);
 -            EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(dis);
 -            ReplayPosition replayPosition = desc.version.metadataIncludesReplayPosition
 -                                          ? ReplayPosition.serializer.deserialize(dis)
 -                                          : ReplayPosition.NONE;
 -            if (!desc.version.metadataIncludesModernReplayPosition)
 -            {
 -                // replay position may be "from the future" thanks to older versions generating them with nanotime.
 -                // make sure we don't omit replaying something that we should.  see CASSANDRA-4782
 -                replayPosition = ReplayPosition.NONE;
 -            }
 -            long minTimestamp = desc.version.tracksMinTimestamp ? dis.readLong() : Long.MIN_VALUE;
 -            long maxTimestamp = desc.version.containsTimestamp() ? dis.readLong() : Long.MAX_VALUE;
 -            if (!desc.version.tracksMaxTimestamp) // see javadoc to Descriptor.containsTimestamp
 -                maxTimestamp = Long.MAX_VALUE;
 -            double compressionRatio = desc.version.hasCompressionRatio
 -                                    ? dis.readDouble()
 -                                    : NO_COMPRESSION_RATIO;
 -            String partitioner = desc.version.hasPartitioner ? dis.readUTF() : null;
 -            int nbAncestors = desc.version.hasAncestors ? dis.readInt() : 0;
 +            EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(in);
 +            EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(in);
 +            ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(in);
 +            long minTimestamp = in.readLong();
 +            long maxTimestamp = in.readLong();
 +            int maxLocalDeletionTime = desc.version.tracksMaxLocalDeletionTime ? in.readInt() : Integer.MAX_VALUE;
 +            double bloomFilterFPChance = desc.version.hasBloomFilterFPChance ? in.readDouble() : NO_BLOOM_FLITER_FP_CHANCE;
 +            double compressionRatio = in.readDouble();
 +            String partitioner = in.readUTF();
 +            int nbAncestors = in.readInt();
              Set<Integer> ancestors = new HashSet<Integer>(nbAncestors);
              for (int i = 0; i < nbAncestors; i++)
 -                ancestors.add(dis.readInt());
 -            StreamingHistogram tombstoneHistogram = desc.version.tracksTombstones
 -                                                   ? StreamingHistogram.serializer.deserialize(dis)
 -                                                   : defaultTombstoneDropTimeHistogram();
 +                ancestors.add(in.readInt());
 +            StreamingHistogram tombstoneHistogram = StreamingHistogram.serializer.deserialize(in);
 +            int sstableLevel = 0;
 +
 +            if (loadSSTableLevel && in.available() > 0)
 +                sstableLevel = in.readInt();
 +
 +            List<ByteBuffer> minColumnNames;
 +            List<ByteBuffer> maxColumnNames;
 +            if (desc.version.tracksMaxMinColumnNames)
 +            {
 +                int colCount = in.readInt();
 +                minColumnNames = new ArrayList<ByteBuffer>(colCount);
 +                for (int i = 0; i < colCount; i++)
 +                {
 +                    minColumnNames.add(ByteBufferUtil.readWithShortLength(in));
 +                }
 +                colCount = in.readInt();
 +                maxColumnNames = new ArrayList<ByteBuffer>(colCount);
 +                for (int i = 0; i < colCount; i++)
 +                {
 +                    maxColumnNames.add(ByteBufferUtil.readWithShortLength(in));
 +                }
 +            }
 +            else
 +            {
 +                minColumnNames = Collections.emptyList();
 +                maxColumnNames = Collections.emptyList();
 +            }
-             return new SSTableMetadata(rowSizes,
+             return Pair.create(new SSTableMetadata(rowSizes,
 -                                                   columnCounts,
 -                                                   replayPosition,
 -                                                   minTimestamp,
 -                                                   maxTimestamp,
 -                                                   compressionRatio,
 -                                                   partitioner,
 -                                                   tombstoneHistogram),
 -                                                   ancestors);
 +                                       columnCounts,
 +                                       replayPosition,
 +                                       minTimestamp,
 +                                       maxTimestamp,
 +                                       maxLocalDeletionTime,
 +                                       bloomFilterFPChance,
 +                                       compressionRatio,
 +                                       partitioner,
-                                        ancestors,
 +                                       tombstoneHistogram,
 +                                       sstableLevel,
 +                                       minColumnNames,
-                                        maxColumnNames);
++                                       maxColumnNames), ancestors);
          }
      }
  }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/07ca445e/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/SSTableReader.java
index b1af7c2,ab58c11..81f83d1
--- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
@@@ -167,10 -165,10 +167,10 @@@ public class SSTableReader extends SSTa
          assert components.contains(Component.DATA) : "Data component is missing for sstable" + descriptor;
          assert components.contains(Component.PRIMARY_INDEX) : "Primary index component is missing for sstable " + descriptor;
  
 -        long start = System.currentTimeMillis();
 +        long start = System.nanoTime();
          logger.info("Opening {} ({} bytes)", descriptor, new File(descriptor.filenameFor(COMPONENT_DATA)).length());
  
-         SSTableMetadata sstableMetadata = SSTableMetadata.serializer.deserialize(descriptor);
+         SSTableMetadata sstableMetadata = SSTableMetadata.serializer.deserialize(descriptor).left;
  
          // Check if sstable is created using same partitioner.
          // Partitioner can be null, which indicates older version of sstable or no stats available.
@@@ -1159,33 -1116,17 +1159,41 @@@
  
      public Set<Integer> getAncestors()
      {
-         return sstableMetadata.ancestors;
+         try
+         {
+             return SSTableMetadata.serializer.deserialize(descriptor).right;
+         }
+         catch (IOException e)
+         {
+             SSTableReader.logOpenException(descriptor, e);
+             return Collections.emptySet();
+         }
      }
  
 +    public int getSSTableLevel()
 +    {
 +        return sstableMetadata.sstableLevel;
 +    }
 +
 +    /**
 +     * Reloads the sstable metadata from disk.
 +     *
 +     * Called after level is changed on sstable, for example if the sstable is dropped to L0
 +     *
 +     * Might be possible to remove in future versions
 +     *
 +     * @throws IOException
 +     */
 +    public void reloadSSTableMetadata() throws IOException
 +    {
-         this.sstableMetadata = SSTableMetadata.serializer.deserialize(descriptor);
++        this.sstableMetadata = SSTableMetadata.serializer.deserialize(descriptor).left;
 +    }
 +
 +    public SSTableMetadata getSSTableMetadata()
 +    {
 +        return sstableMetadata;
 +    }
 +
      public RandomAccessReader openDataReader(RateLimiter limiter)
      {
          assert limiter != null;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/07ca445e/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
index 2f54e1a,54e154c..1d69bd0
--- a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
@@@ -303,9 -327,9 +303,9 @@@ public class SSTableWriter extends SSTa
          // main data, close will truncate if necessary
          dataFile.close();
          // write sstable statistics
 -        SSTableMetadata sstableMetadata = sstableMetadataCollector.finalizeMetadata(partitioner.getClass().getCanonicalName());
 +        SSTableMetadata sstableMetadata = sstableMetadataCollector.finalizeMetadata(partitioner.getClass().getCanonicalName(),
 +                                                                                    metadata.getBloomFilterFpChance());
-         writeMetadata(descriptor, sstableMetadata);
+         writeMetadata(descriptor, sstableMetadata, sstableMetadataCollector.ancestors);
 -        maybeWriteDigest();
  
          // save the table of components
          SSTable.appendTOC(descriptor, components);
@@@ -335,7 -359,29 +335,7 @@@
          return sstable;
      }
  
-     private static void writeMetadata(Descriptor desc, SSTableMetadata sstableMetadata)
 -    private void maybeWriteDigest()
 -    {
 -        byte[] digest = dataFile.digest();
 -        if (digest == null)
 -            return;
 -
 -        SequentialWriter out = SequentialWriter.open(new File(descriptor.filenameFor(SSTable.COMPONENT_DIGEST)), true);
 -        // Writting output compatible with sha1sum
 -        Descriptor newdesc = descriptor.asTemporary(false);
 -        String[] tmp = newdesc.filenameFor(SSTable.COMPONENT_DATA).split(Pattern.quote(File.separator));
 -        String dataFileName = tmp[tmp.length - 1];
 -        try
 -        {
 -            out.write(String.format("%s  %s", Hex.bytesToHex(digest), dataFileName).getBytes());
 -        }
 -        catch (ClosedChannelException e)
 -        {
 -            throw new AssertionError(); // can't happen.
 -        }
 -        out.close();
 -    }
 -
 -    private static void writeMetadata(Descriptor desc, SSTableMetadata sstableMetadata, Set<Integer> ancestors)
++    private static void writeMetadata(Descriptor desc, SSTableMetadata sstableMetadata,  Set<Integer> ancestors)
      {
          SequentialWriter out = SequentialWriter.open(new File(desc.filenameFor(SSTable.COMPONENT_STATS)), true);
          try

http://git-wip-us.apache.org/repos/asf/cassandra/blob/07ca445e/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
index 7598750,0000000..999f440
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
+++ b/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
@@@ -1,78 -1,0 +1,79 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.cassandra.tools;
 +
 +import java.io.IOException;
 +import java.io.PrintStream;
 +import java.util.Map;
 +import java.util.Set;
 +
 +import org.apache.cassandra.db.Directories;
 +import org.apache.cassandra.db.compaction.LeveledManifest;
 +import org.apache.cassandra.io.sstable.Component;
 +import org.apache.cassandra.io.sstable.Descriptor;
 +import org.apache.cassandra.io.sstable.SSTableMetadata;
++import org.apache.cassandra.utils.Pair;
 +
 +/**
 + * Reset level to 0 on a given set of sstables
 + */
 +public class SSTableLevelResetter
 +{
 +    /**
 +     * @param args a list of sstables whose metadata we are changing
 +     */
 +    public static void main(String[] args) throws IOException
 +    {
 +        PrintStream out = System.out;
 +        if (args.length == 0)
 +        {
 +            out.println("This command should be run with Cassandra stopped!");
 +            out.println("Usage: sstablelevelreset <keyspace> <columnfamily>");
 +            System.exit(1);
 +        }
 +
 +        if (!args[0].equals("--really-reset") || args.length != 3)
 +        {
 +            out.println("This command should be run with Cassandra stopped, otherwise you will get very strange behavior");
 +            out.println("Verify that Cassandra is not running and then execute the command like this:");
 +            out.println("Usage: sstablelevelreset --really-reset <keyspace> <columnfamily>");
 +            System.exit(1);
 +        }
 +
 +        String keyspace = args[1];
 +        String columnfamily = args[2];
 +        Directories directories = Directories.create(keyspace, columnfamily);
 +        boolean foundSSTable = false;
 +        for (Map.Entry<Descriptor, Set<Component>> sstable : directories.sstableLister().list().entrySet())
 +        {
 +            if (sstable.getValue().contains(Component.STATS))
 +            {
 +                foundSSTable = true;
 +                Descriptor descriptor = sstable.getKey();
-                 SSTableMetadata metadata = SSTableMetadata.serializer.deserialize(descriptor);
-                 out.println("Changing level from " + metadata.sstableLevel + " to 0 on " + descriptor.filenameFor(Component.DATA));
++                Pair<SSTableMetadata, Set<Integer>> metadata = SSTableMetadata.serializer.deserialize(descriptor);
++                out.println("Changing level from " + metadata.left.sstableLevel + " to 0 on " + descriptor.filenameFor(Component.DATA));
 +                LeveledManifest.mutateLevel(metadata, descriptor, descriptor.filenameFor(Component.STATS), 0);
 +            }
 +        }
 +
 +        if (!foundSSTable)
 +        {
 +            out.println("Found no sstables, did you give the correct keyspace/columnfamily?");
 +        }
 +    }
 +}

http://git-wip-us.apache.org/repos/asf/cassandra/blob/07ca445e/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/07ca445e/test/unit/org/apache/cassandra/db/compaction/LegacyLeveledManifestTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/compaction/LegacyLeveledManifestTest.java
index 76fb480,0000000..d8e8af0
mode 100644,000000..100644
--- a/test/unit/org/apache/cassandra/db/compaction/LegacyLeveledManifestTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/LegacyLeveledManifestTest.java
@@@ -1,114 -1,0 +1,117 @@@
 +package org.apache.cassandra.db.compaction;
 +/*
 + * 
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + * 
 + *   http://www.apache.org/licenses/LICENSE-2.0
 + * 
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + * 
 + */
 +
 +
 +import java.io.File;
 +import java.io.IOException;
 +import java.util.HashMap;
 +import java.util.Map;
++import java.util.Set;
 +import org.junit.After;
 +import org.junit.Before;
 +import org.junit.Test;
 +
 +import org.apache.cassandra.db.Directories;
 +import org.apache.cassandra.io.sstable.Descriptor;
 +import org.apache.cassandra.io.sstable.SSTableMetadata;
 +import org.apache.cassandra.io.util.FileUtils;
++import org.apache.cassandra.utils.Pair;
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertTrue;
 +import static org.apache.cassandra.db.compaction.LegacyLeveledManifestTestHelper.*;
 +
 +public class LegacyLeveledManifestTest
 +{
 +    private final static String LEGACY_VERSION = "ic";
 +
 +    private File destDir;
 +    @Before
 +    public void setup()
 +    {
 +        destDir = Directories.create(KS, CF).getDirectoryForNewSSTables(0);
 +        FileUtils.createDirectory(destDir);
 +        for (File srcFile : getLegacySSTableDir(LEGACY_VERSION).listFiles())
 +        {
 +            File destFile = new File(destDir, srcFile.getName());
 +            FileUtils.createHardLink(srcFile,destFile);
 +            assert destFile.exists() : destFile.getAbsoluteFile();
 +        }
 +    }
 +    @After
 +    public void tearDown()
 +    {
 +        FileUtils.deleteRecursive(destDir);
 +    }
 +
 +    @Test
 +    public void migrateTest() throws IOException
 +    {
 +        assertTrue(LegacyLeveledManifest.manifestNeedsMigration(KS, CF));
 +    }
 +
 +    @Test
 +    public void doMigrationTest() throws IOException, InterruptedException
 +    {
 +        LegacyLeveledManifest.migrateManifests(KS, CF);
 +
 +        for (int i = 0; i <= 2; i++)
 +        {
 +            Descriptor descriptor = Descriptor.fromFilename(destDir+File.separator+KS+"-"+CF+"-"+LEGACY_VERSION+"-"+i+"-Statistics.db");
-             SSTableMetadata metadata = SSTableMetadata.serializer.deserialize(descriptor);
++            SSTableMetadata metadata = SSTableMetadata.serializer.deserialize(descriptor).left;
 +            assertEquals(metadata.sstableLevel, i);
 +        }
 +    }
 +
 +    /**
 +     * Validate that the rewritten stats file is the same as the original one.
 +     * @throws IOException
 +     */
 +    @Test
 +    public void validateSSTableMetadataTest() throws IOException
 +    {
-         Map<Descriptor, SSTableMetadata> beforeMigration = new HashMap<Descriptor, SSTableMetadata>();
++        Map<Descriptor, Pair<SSTableMetadata, Set<Integer>>> beforeMigration = new HashMap<>();
 +        for (int i = 0; i <= 2; i++)
 +        {
 +            Descriptor descriptor = Descriptor.fromFilename(destDir+File.separator+KS+"-"+CF+"-"+LEGACY_VERSION+"-"+i+"-Statistics.db");
 +            beforeMigration.put(descriptor, SSTableMetadata.serializer.deserialize(descriptor, false));
 +        }
 +
 +        LegacyLeveledManifest.migrateManifests(KS, CF);
 +
-         for (Map.Entry<Descriptor, SSTableMetadata> entry : beforeMigration.entrySet())
++        for (Map.Entry<Descriptor, Pair<SSTableMetadata, Set<Integer>>> entry : beforeMigration.entrySet())
 +        {
-             SSTableMetadata newMetadata = SSTableMetadata.serializer.deserialize(entry.getKey());
-             SSTableMetadata oldMetadata = entry.getValue();
++            Pair<SSTableMetadata, Set<Integer>> newMetaPair = SSTableMetadata.serializer.deserialize(entry.getKey());
++            SSTableMetadata newMetadata = newMetaPair.left;
++            SSTableMetadata oldMetadata = entry.getValue().left;
 +            assertEquals(newMetadata.estimatedRowSize, oldMetadata.estimatedRowSize);
 +            assertEquals(newMetadata.estimatedColumnCount, oldMetadata.estimatedColumnCount);
 +            assertEquals(newMetadata.replayPosition, oldMetadata.replayPosition);
 +            assertEquals(newMetadata.minTimestamp, oldMetadata.minTimestamp);
 +            assertEquals(newMetadata.maxTimestamp, oldMetadata.maxTimestamp);
 +            assertEquals(newMetadata.compressionRatio, oldMetadata.compressionRatio, 0.01);
 +            assertEquals(newMetadata.partitioner, oldMetadata.partitioner);
-             assertEquals(newMetadata.ancestors, oldMetadata.ancestors);
 +            assertEquals(newMetadata.estimatedTombstoneDropTime, oldMetadata.estimatedTombstoneDropTime);
++            assertEquals(entry.getValue().right, newMetaPair.right);
 +        }
 +    }
 +
 +}

http://git-wip-us.apache.org/repos/asf/cassandra/blob/07ca445e/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
index 3aac83e,e073c41..b60f6d9
--- a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
@@@ -35,19 -29,15 +35,20 @@@ import org.apache.cassandra.Util
  import org.apache.cassandra.db.ColumnFamilyStore;
  import org.apache.cassandra.db.DecoratedKey;
  import org.apache.cassandra.db.RowMutation;
 -import org.apache.cassandra.db.Table;
 -import org.apache.cassandra.db.filter.QueryPath;
 +import org.apache.cassandra.db.Keyspace;
  import org.apache.cassandra.dht.Range;
  import org.apache.cassandra.dht.Token;
 +import org.apache.cassandra.io.sstable.Component;
  import org.apache.cassandra.io.sstable.SSTable;
  import org.apache.cassandra.io.sstable.SSTableReader;
 -import org.apache.cassandra.service.AntiEntropyService;
 +import org.apache.cassandra.repair.RepairJobDesc;
 +import org.apache.cassandra.repair.Validator;
  import org.apache.cassandra.utils.ByteBufferUtil;
  import org.apache.cassandra.utils.FBUtilities;
++import org.apache.cassandra.utils.Pair;
 +
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertTrue;
  
  @RunWith(OrderedJUnit4ClassRunner.class)
  public class LeveledCompactionStrategyTest extends SchemaLoader
@@@ -146,55 -128,4 +147,55 @@@
          // scanner.getCurrentPosition should be equal to total bytes of L1 sstables
          assert scanner.getCurrentPosition() == SSTable.getTotalBytes(sstables);
      }
 +
 +    @Test
 +    public void testMutateLevel() throws Exception
 +    {
 +        String ksname = "Keyspace1";
 +        String cfname = "StandardLeveled";
 +        Keyspace keyspace = Keyspace.open(ksname);
 +        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
 +
 +        ByteBuffer value = ByteBuffer.wrap(new byte[100 * 1024]); // 100 KB value, make it easy to have multiple files
 +
 +        // Enough data to have a level 1 and 2
 +        int rows = 20;
 +        int columns = 10;
 +
 +        // Adds enough data to trigger multiple sstable per level
 +        for (int r = 0; r < rows; r++)
 +        {
 +            DecoratedKey key = Util.dk(String.valueOf(r));
 +            RowMutation rm = new RowMutation(ksname, key.key);
 +            for (int c = 0; c < columns; c++)
 +            {
 +                rm.add(cfname, ByteBufferUtil.bytes("column" + c), value, 0);
 +            }
 +            rm.apply();
 +            cfs.forceBlockingFlush();
 +        }
 +        waitForLeveling(cfs);
 +        cfs.forceBlockingFlush();
 +        LeveledCompactionStrategy strategy = (LeveledCompactionStrategy) cfs.getCompactionStrategy();
 +        cfs.disableAutoCompaction();
 +
 +        while(CompactionManager.instance.isCompacting(Arrays.asList(cfs)))
 +            Thread.sleep(100);
 +
 +        for (SSTableReader s : cfs.getSSTables())
 +        {
 +            assertTrue(s.getSSTableLevel() != 6);
 +            strategy.manifest.remove(s);
-             LeveledManifest.mutateLevel(s.getSSTableMetadata(), s.descriptor, s.descriptor.filenameFor(Component.STATS), 6);
++            LeveledManifest.mutateLevel(Pair.create(s.getSSTableMetadata(), s.getAncestors()), s.descriptor, s.descriptor.filenameFor(Component.STATS), 6);
 +            s.reloadSSTableMetadata();
 +            strategy.manifest.add(s);
 +        }
 +        // verify that all sstables in the changed set is level 6
 +        for (SSTableReader s : cfs.getSSTables())
 +            assertEquals(6, s.getSSTableLevel());
 +
 +        int[] levels = strategy.manifest.getAllLevelSize();
 +        // verify that the manifest has correct amount of sstables
 +        assertEquals(cfs.getSSTables().size(), levels[6]);
 +    }
  }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/07ca445e/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java
index 889f468,3e8ef42..13b2f26
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataSerializerTest.java
@@@ -28,9 -31,9 +31,10 @@@ import java.util.Set
  import org.junit.Test;
  
  import org.apache.cassandra.db.commitlog.ReplayPosition;
 +import org.apache.cassandra.db.marshal.BytesType;
  import org.apache.cassandra.dht.RandomPartitioner;
  import org.apache.cassandra.utils.EstimatedHistogram;
+ import org.apache.cassandra.utils.Pair;
  
  public class SSTableMetadataSerializerTest
  {
@@@ -53,17 -56,19 +57,21 @@@
                                                               .replayPosition(rp);
          collector.updateMinTimestamp(minTimestamp);
          collector.updateMaxTimestamp(maxTimestamp);
 -        SSTableMetadata originalMetadata = collector.finalizeMetadata(RandomPartitioner.class.getCanonicalName());
 +        SSTableMetadata originalMetadata = collector.finalizeMetadata(RandomPartitioner.class.getCanonicalName(), 0.1);
  
          ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
 -        DataOutputStream dos = new DataOutputStream(byteOutput);
 +        DataOutputStream out = new DataOutputStream(byteOutput);
++        
+         Set<Integer> ancestors = new HashSet<Integer>();
+         ancestors.addAll(Arrays.asList(1,2,3,4));
 -        SSTableMetadata.serializer.serialize(originalMetadata, ancestors, dos);
 +
-         SSTableMetadata.serializer.serialize(originalMetadata, out);
++        SSTableMetadata.serializer.serialize(originalMetadata, ancestors, out);
  
          ByteArrayInputStream byteInput = new ByteArrayInputStream(byteOutput.toByteArray());
 -        DataInputStream dis = new DataInputStream(byteInput);
 +        DataInputStream in = new DataInputStream(byteInput);
          Descriptor desc = new Descriptor(Descriptor.Version.CURRENT, new File("."), "", "", 0, false);
-         SSTableMetadata stats = SSTableMetadata.serializer.deserialize(in, desc);
 -        Pair<SSTableMetadata, Set<Integer>> statsPair = SSTableMetadata.serializer.deserialize(dis, desc);
++        Pair<SSTableMetadata, Set<Integer>> statsPair = SSTableMetadata.serializer.deserialize(in, desc);
+         SSTableMetadata stats = statsPair.left;
  
          assert stats.estimatedRowSize.equals(originalMetadata.estimatedRowSize);
          assert stats.estimatedRowSize.equals(rowSizes);
@@@ -75,7 -80,7 +83,8 @@@
          assert stats.maxTimestamp == maxTimestamp;
          assert stats.minTimestamp == originalMetadata.minTimestamp;
          assert stats.maxTimestamp == originalMetadata.maxTimestamp;
 +        assert stats.bloomFilterFPChance == originalMetadata.bloomFilterFPChance;
          assert RandomPartitioner.class.getCanonicalName().equals(stats.partitioner);
+         assert ancestors.equals(statsPair.right);
      }
  }


Mime
View raw message