cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject [2/10] git commit: merge from 1.0
Date Thu, 03 May 2012 23:35:43 GMT
merge from 1.0


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/af1dc0f5
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/af1dc0f5
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/af1dc0f5

Branch: refs/heads/trunk
Commit: af1dc0f56a560ddee1e98bfea677bf534d38cc39
Parents: 249456b c1148ce
Author: Jonathan Ellis <jbellis@apache.org>
Authored: Thu May 3 18:31:29 2012 -0500
Committer: Jonathan Ellis <jbellis@apache.org>
Committed: Thu May 3 18:31:29 2012 -0500

----------------------------------------------------------------------
 CHANGES.txt                                        |    9 +++-
 src/java/org/apache/cassandra/db/Directories.java  |   33 ++++----------
 .../cassandra/db/compaction/CompactionTask.java    |   20 ++-------
 .../apache/cassandra/io/sstable/Descriptor.java    |   15 ++++++-
 .../cassandra/io/sstable/SSTableMetadata.java      |    4 +-
 5 files changed, 38 insertions(+), 43 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/af1dc0f5/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index b13d69b,1c1c184..1b86758
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,121 -1,19 +1,128 @@@
 -1.0.10
 +1.1.1-dev
 + * include tombstone size in ColumnFamily.size, which can prevent OOM
 +   during sudden mass delete operations (CASSANDRA-3741)
 + * Open 1 sstableScanner per level for leveled compaction (CASSANDRA-4142)
 + * Optimize reads when row deletion timestamps allow us to restrict
 +   the set of sstables we check (CASSANDRA-4116)
 + * incremental repair by token range (CASSANDRA-3912)
 + * streaming commitlog backup + pitr (CASSANDRA-3690)
 + * avoid generating redundant compaction tasks during streaming
 +   (CASSANDRA-4174)
 + * add -cf option to nodetool snapshot, and takeColumnFamilySnapshot to
 +   StorageService mbean (CASSANDRA-556)
 + * optimize cleanup to drop entire sstables where possible (CASSANDRA-4079)
 + * optimize truncate when autosnapshot is disabled (CASSANDRA-4153)
 + * add support for commitlog archiving and point-in-time recovery
 +   (CASSANDRA-3647)
 + * update caches to use byte[] keys to reduce memory overhead (CASSANDRA-3966)
 + * add column limit to cli (CASSANDRA-3012, 4098)
 + * clean up and optimize DataOutputBuffer, used by CQL compression and
 +   CompositeType (CASSANDRA-4072)
 + * optimize commitlog checksumming (CASSANDRA-3610)
 + * identify and blacklist corrupted SSTables from future compactions 
 +   (CASSANDRA-2261)
 + * Move CfDef and KsDef validation out of thrift (CASSANDRA-4037)
 + * Expose repairing by a user provided range (CASSANDRA-3912)
 + * Add way to force the cassandra-cli to refresh it's schema (CASSANDRA-4052)
 + * Avoids having replicate on write tasks stacking up at CL.ONE (CASSANDRA-2889)
 + * (cql3) Fix order by for reversed queries (CASSANDRA-4160)
 + * (cql3) Add ReversedType support (CASSANDRA-4004)
 + * (cql3) Add timeuuid type (CASSANDRA-4194)
 + * (cql3) Minor fixes (CASSANDRA-4185)
 + * (cql3) Fix prepared statement in BATCH (CASSANDRA-4202)
 + * (cql3) Reduce the list of reserved keywords (CASSANDRA-4186)
 + * (cql3) Move max/min compaction thresholds to compaction strategy options
 +   (CASSANDRA-4187)
 +Merged from 1.0:
 + * Fix super columns bug where cache is not updated (CASSANDRA-4190)
   * fix maxTimestamp to include row tombstones (CASSANDRA-4116)
 + * (CLI) properly handle quotes in create/update keyspace commands (CASSANDRA-4129)
 + * Avoids possible deadlock during bootstrap (CASSANDRA-4159)
 + * fix stress tool that hangs forever on timeout or error (CASSANDRA-4128)
 + * Fix super columns bug where cache is not updated (CASSANDRA-4190)
 + * stress tool to return appropriate exit code on failure (CASSANDRA-4188)
++ * fix compaction NPE when out of disk space and assertions disabled
++   (CASSANDRA-3985)
 +
 +
 +1.1.0-final
 + * average a reduced liveRatio estimate with the previous one (CASSANDRA-4065)
 + * Allow KS and CF names up to 48 characters (CASSANDRA-4157)
 + * fix stress build (CASSANDRA-4140)
 + * add time remaining estimate to nodetool compactionstats (CASSANDRA-4167)
 + * (cql) fix NPE in cql3 ALTER TABLE (CASSANDRA-4163)
 + * (cql) Add support for CL.TWO and CL.THREE in CQL (CASSANDRA-4156)
 + * (cql) Fix type in CQL3 ALTER TABLE preventing update (CASSANDRA-4170)
 + * (cql) Throw invalid exception from CQL3 on obsolete options (CASSANDRA-4171)
 + * (cqlsh) fix recognizing uppercase SELECT keyword (CASSANDRA-4161)
 + * Pig: wide row support (CASSANDRA-3909)
 +Merged from 1.0:
   * avoid streaming empty files with bulk loader if sstablewriter errors out
     (CASSANDRA-3946)
 +
 +
 +1.1-rc1
 + * Include stress tool in binary builds (CASSANDRA-4103)
 + * (Hadoop) fix wide row iteration when last row read was deleted
 +   (CASSANDRA-4154)
 + * fix read_repair_chance to really default to 0.1 in the cli (CASSANDRA-4114)
 + * Adds caching and bloomFilterFpChange to CQL options (CASSANDRA-4042)
 + * Adds posibility to autoconfigure size of the KeyCache (CASSANDRA-4087)
 + * fix KEYS index from skipping results (CASSANDRA-3996)
 + * Remove sliced_buffer_size_in_kb dead option (CASSANDRA-4076)
 + * make loadNewSStable preserve sstable version (CASSANDRA-4077)
 + * Respect 1.0 cache settings as much as possible when upgrading 
 +   (CASSANDRA-4088)
 + * relax path length requirement for sstable files when upgrading on 
 +   non-Windows platforms (CASSANDRA-4110)
 + * fix terminination of the stress.java when errors were encountered
 +   (CASSANDRA-4128)
 + * Move CfDef and KsDef validation out of thrift (CASSANDRA-4037)
 + * Fix get_paged_slice (CASSANDRA-4136)
 + * CQL3: Support slice with exclusive start and stop (CASSANDRA-3785)
 +Merged from 1.0:
   * support PropertyFileSnitch in bulk loader (CASSANDRA-4145)
   * add auto_snapshot option allowing disabling snapshot before drop/truncate
     (CASSANDRA-3710)
   * allow short snitch names (CASSANDRA-4130)
 +
 +
 +1.1-beta2
 + * rename loaded sstables to avoid conflicts with local snapshots
 +   (CASSANDRA-3967)
 + * start hint replay as soon as FD notifies that the target is back up
 +   (CASSANDRA-3958)
 + * avoid unproductive deserializing of cached rows during compaction
 +   (CASSANDRA-3921)
 + * fix concurrency issues with CQL keyspace creation (CASSANDRA-3903)
 + * Show Effective Owership via Nodetool ring <keyspace> (CASSANDRA-3412)
 + * Update ORDER BY syntax for CQL3 (CASSANDRA-3925)
 + * Fix BulkRecordWriter to not throw NPE if reducer gets no map data from Hadoop (CASSANDRA-3944)
 + * Fix bug with counters in super columns (CASSANDRA-3821)
 + * Remove deprecated merge_shard_chance (CASSANDRA-3940)
 + * add a convenient way to reset a node's schema (CASSANDRA-2963)
 + * fix for intermittent SchemaDisagreementException (CASSANDRA-3884)
 + * CLI `list <CF>` to limit number of columns and their order (CASSANDRA-3012)
 + * ignore deprecated KsDef/CfDef/ColumnDef fields in native schema (CASSANDRA-3963)
 + * CLI to report when unsupported column_metadata pair was given (CASSANDRA-3959)
 + * reincarnate removed and deprecated KsDef/CfDef attributes (CASSANDRA-3953)
 + * Fix race between writes and read for cache (CASSANDRA-3862)
 + * perform static initialization of StorageProxy on start-up (CASSANDRA-3797)
 + * support trickling fsync() on writes (CASSANDRA-3950)
 + * expose counters for unavailable/timeout exceptions given to thrift clients (CASSANDRA-3671)
 + * avoid quadratic startup time in LeveledManifest (CASSANDRA-3952)
 + * Add type information to new schema_ columnfamilies and remove thrift
 +   serialization for schema (CASSANDRA-3792)
 + * add missing column validator options to the CLI help (CASSANDRA-3926)
 + * skip reading saved key cache if CF's caching strategy is NONE or ROWS_ONLY (CASSANDRA-3954)
 + * Unify migration code (CASSANDRA-4017)
 +Merged from 1.0:
   * cqlsh: guess correct version of Python for Arch Linux (CASSANDRA-4090)
+  * (CLI) properly handle quotes in create/update keyspace commands (CASSANDRA-4129)
+  * Avoids possible deadlock during bootstrap (CASSANDRA-4159)
+  * fix stress tool that hangs forever on timeout or error (CASSANDRA-4128)
+  * Fix super columns bug where cache is not updated (CASSANDRA-4190)
+  * stress tool to return appropriate exit code on failure (CASSANDRA-4188)
 - * fix compaction NPE when out of disk space and assertions disabled
 -   (CASSANDRA-3985)
  
  
  1.0.9

http://git-wip-us.apache.org/repos/asf/cassandra/blob/af1dc0f5/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/Directories.java
index 6d16559,0000000..f023c8c
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@@ -1,584 -1,0 +1,571 @@@
 +/**
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.cassandra.db;
 +
 +import java.io.File;
 +import java.io.FileFilter;
 +import java.io.IOError;
 +import java.io.IOException;
 +import java.util.*;
 +
 +import org.apache.commons.lang.StringUtils;
 +import com.google.common.collect.ImmutableMap;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +import org.apache.cassandra.config.*;
 +import org.apache.cassandra.db.compaction.LeveledManifest;
 +import org.apache.cassandra.io.util.FileUtils;
 +import org.apache.cassandra.io.util.MmappedSegmentedFile;
 +import org.apache.cassandra.io.sstable.*;
 +import org.apache.cassandra.service.StorageService;
 +import org.apache.cassandra.utils.CLibrary;
 +import org.apache.cassandra.utils.Pair;
 +
 +/**
 + * Encapsulate handling of paths to the data files.
 + *
 + * The directory layout is the following:
 + *   /<path_to_data_dir>/ks/cf1/ks-cf1-hb-1-Data.db
 + *                         /cf2/ks-cf2-hb-1-Data.db
 + *                         ...
 + *
 + * In addition, more that one 'root' data directory can be specified so that
 + * <path_to_data_dir> potentially represents multiple locations.
 + * Note that in the case of multiple locations, the manifest for the leveled
 + * compaction is only in one of the location.
 + *
 + * Snapshots (resp. backups) are always created along the sstables thare are
 + * snapshoted (resp. backuped) but inside a subdirectory named 'snapshots'
 + * (resp. backups) (and snapshots are furter inside a subdirectory of the name
 + * of the snapshot).
 + *
 + * This class abstracts all those details from the rest of the code.
 + */
 +public class Directories
 +{
 +    private static Logger logger = LoggerFactory.getLogger(Directories.class);
 +
 +    public static final String BACKUPS_SUBDIR = "backups";
 +    public static final String SNAPSHOT_SUBDIR = "snapshots";
 +    public static final char SECONDARY_INDEX_NAME_SEPARATOR = '.';
 +
 +    public static final File[] dataFileLocations;
 +    static
 +    {
 +        String[] locations = DatabaseDescriptor.getAllDataFileLocations();
 +        dataFileLocations = new File[locations.length];
 +        for (int i = 0; i < locations.length; ++i)
 +            dataFileLocations[i] = new File(locations[i]);
 +    }
 +
 +    private final String tablename;
 +    private final String cfname;
 +    private final File[] sstableDirectories;
 +
 +    public static Directories create(String tablename, String cfname)
 +    {
 +        int idx = cfname.indexOf(SECONDARY_INDEX_NAME_SEPARATOR);
 +        if (idx > 0)
 +            // secondary index, goes in the same directory than the base cf
 +            return new Directories(tablename, cfname, cfname.substring(0, idx));
 +        else
 +            return new Directories(tablename, cfname, cfname);
 +    }
 +
 +    private Directories(String tablename, String cfname, String directoryName)
 +    {
 +        this.tablename = tablename;
 +        this.cfname = cfname;
 +        this.sstableDirectories = new File[dataFileLocations.length];
 +        for (int i = 0; i < dataFileLocations.length; ++i)
 +            sstableDirectories[i] = new File(dataFileLocations[i], join(tablename, directoryName));
 +
 +        if (!StorageService.instance.isClientMode())
 +        {
 +            try
 +            {
 +                for (File dir : sstableDirectories)
 +                    FileUtils.createDirectory(dir);
 +            }
 +            catch (IOException e)
 +            {
 +                throw new IOError(e);
 +            }
 +        }
 +    }
 +
 +    public File getDirectoryForNewSSTables(long estimatedSize)
 +    {
-         return getDirectoryForNewSSTables(estimatedSize, true);
-     }
++        File path = getLocationWithMaximumAvailableSpace(estimatedSize);
 +
-     public File getDirectoryForNewSSTables(long estimatedSize, boolean ensureFreeSpace)
-     {
-         File path = getLocationWithMaximumAvailableSpace(estimatedSize, ensureFreeSpace);
 +        // Requesting GC has a chance to free space only if we're using mmap and a non SUN
jvm
 +        if (path == null
-          && (DatabaseDescriptor.getDiskAccessMode() == Config.DiskAccessMode.mmap
|| DatabaseDescriptor.getIndexAccessMode() == Config.DiskAccessMode.mmap)
-          && !MmappedSegmentedFile.isCleanerAvailable())
++            && (DatabaseDescriptor.getDiskAccessMode() == Config.DiskAccessMode.mmap
|| DatabaseDescriptor.getIndexAccessMode() == Config.DiskAccessMode.mmap)
++            && !MmappedSegmentedFile.isCleanerAvailable())
 +        {
++            logger.info("Forcing GC to free up disk space.  Upgrade to the Oracle JVM to
avoid this");
 +            StorageService.instance.requestGC();
 +            // retry after GCing has forced unmap of compacted SSTables so they can be deleted
 +            // Note: GCInspector will do this already, but only sun JVM supports GCInspector
so far
 +            SSTableDeletingTask.rescheduleFailedTasks();
 +            try
 +            {
 +                Thread.sleep(10000);
 +            }
 +            catch (InterruptedException e)
 +            {
 +                throw new AssertionError(e);
 +            }
-             path = getLocationWithMaximumAvailableSpace(estimatedSize, ensureFreeSpace);
++            path = getLocationWithMaximumAvailableSpace(estimatedSize);
 +        }
++
 +        return path;
 +    }
 +
 +    /*
 +     * Loop through all the disks to see which disk has the max free space
 +     * return the disk with max free space for compactions. If the size of the expected
 +     * compacted file is greater than the max disk space available return null, we cannot
 +     * do compaction in this case.
 +     */
-     public File getLocationWithMaximumAvailableSpace(long estimatedSize, boolean ensureFreeSpace)
++    public File getLocationWithMaximumAvailableSpace(long estimatedSize)
 +    {
 +        long maxFreeDisk = 0;
 +        File maxLocation = null;
 +
 +        for (File dir : sstableDirectories)
 +        {
 +            if (maxFreeDisk < dir.getUsableSpace())
 +            {
 +                maxFreeDisk = dir.getUsableSpace();
 +                maxLocation = dir;
 +            }
 +        }
-         logger.debug(String.format("expected data files size is %d; largest free partition
(%s) has %d bytes free",
-                                    estimatedSize,
-                                    maxLocation,
-                                    maxFreeDisk));
- 
 +        // Load factor of 0.9 we do not want to use the entire disk that is too risky.
 +        maxFreeDisk = (long) (0.9 * maxFreeDisk);
++        logger.debug(String.format("expected data files size is %d; largest free partition
(%s) has %d bytes free",
++                                   estimatedSize, maxLocation, maxFreeDisk));
 +
-         if (!ensureFreeSpace || estimatedSize < maxFreeDisk)
-         {
-             if (estimatedSize >= maxFreeDisk)
-                 logger.warn(String.format("Data file location %s only has %d free, estimated
size is %d",
-                                           maxLocation,
-                                           maxFreeDisk,
-                                           estimatedSize));
 +
++        if (estimatedSize < maxFreeDisk)
 +            return maxLocation;
-         }
- 
 +        return null;
 +    }
 +
 +    public static File getSnapshotDirectory(Descriptor desc, String snapshotName)
 +    {
 +        return getOrCreate(desc.directory, SNAPSHOT_SUBDIR, snapshotName);
 +    }
 +
 +    public static File getBackupsDirectory(Descriptor desc)
 +    {
 +        return getOrCreate(desc.directory, BACKUPS_SUBDIR);
 +    }
 +
 +    public SSTableLister sstableLister()
 +    {
 +        return new SSTableLister();
 +    }
 +
 +    public class SSTableLister
 +    {
 +        private boolean skipCompacted;
 +        private boolean skipTemporary;
 +        private boolean includeBackups;
 +        private boolean onlyBackups;
 +        private int nbFiles;
 +        private final Map<Descriptor, Set<Component>> components = new HashMap<Descriptor,
Set<Component>>();
 +        private boolean filtered;
 +        private String snapshotName;
 +
 +        public SSTableLister skipCompacted(boolean b)
 +        {
 +            if (filtered)
 +                throw new IllegalStateException("list() has already been called");
 +            skipCompacted = b;
 +            return this;
 +        }
 +
 +        public SSTableLister skipTemporary(boolean b)
 +        {
 +            if (filtered)
 +                throw new IllegalStateException("list() has already been called");
 +            skipTemporary = b;
 +            return this;
 +        }
 +
 +        public SSTableLister includeBackups(boolean b)
 +        {
 +            if (filtered)
 +                throw new IllegalStateException("list() has already been called");
 +            includeBackups = b;
 +            return this;
 +        }
 +
 +        public SSTableLister onlyBackups(boolean b)
 +        {
 +            if (filtered)
 +                throw new IllegalStateException("list() has already been called");
 +            onlyBackups = b;
 +            includeBackups = b;
 +            return this;
 +        }
 +
 +        public SSTableLister snapshots(String sn)
 +        {
 +            if (filtered)
 +                throw new IllegalStateException("list() has already been called");
 +            snapshotName = sn;
 +            return this;
 +        }
 +
 +        public Map<Descriptor, Set<Component>> list()
 +        {
 +            filter();
 +            return ImmutableMap.copyOf(components);
 +        }
 +
 +        public List<File> listFiles()
 +        {
 +            filter();
 +            List<File> l = new ArrayList<File>(nbFiles);
 +            for (Map.Entry<Descriptor, Set<Component>> entry : components.entrySet())
 +            {
 +                for (Component c : entry.getValue())
 +                {
 +                    l.add(new File(entry.getKey().filenameFor(c)));
 +                }
 +            }
 +            return l;
 +        }
 +
 +        private void filter()
 +        {
 +            if (filtered)
 +                return;
 +
 +            for (File location : sstableDirectories)
 +            {
 +                if (snapshotName != null)
 +                {
 +                    new File(location, join(SNAPSHOT_SUBDIR, snapshotName)).listFiles(getFilter());
 +                    continue;
 +                }
 +
 +                if (!onlyBackups)
 +                    location.listFiles(getFilter());
 +
 +                if (includeBackups)
 +                    new File(location, BACKUPS_SUBDIR).listFiles(getFilter());
 +            }
 +            filtered = true;
 +        }
 +
 +        private FileFilter getFilter()
 +        {
 +            // Note: the prefix needs to include cfname + separator to distinguish between
a cfs and it's secondary indexes
 +            final String sstablePrefix = tablename + Component.separator + cfname + Component.separator;
 +            return new FileFilter()
 +            {
 +                // This function always return false since accepts adds to the components
map
 +                public boolean accept(File file)
 +                {
 +                    // we are only interested in the SSTable files that belong to the specific
ColumnFamily
 +                    if (file.isDirectory() || !file.getName().startsWith(sstablePrefix))
 +                        return false;
 +
 +                    Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file.getParentFile(),
file.getName());
 +                    if (pair == null)
 +                        return false;
 +
 +                    if (skipCompacted && new File(pair.left.filenameFor(Component.COMPACTED_MARKER)).exists())
 +                        return false;
 +                    if (skipTemporary && pair.left.temporary)
 +                        return false;
 +
 +                    Set<Component> previous = components.get(pair.left);
 +                    if (previous == null)
 +                    {
 +                        previous = new HashSet<Component>();
 +                        components.put(pair.left, previous);
 +                    }
 +                    previous.add(pair.right);
 +                    nbFiles++;
 +                    return false;
 +                }
 +            };
 +        }
 +    }
 +
 +    public File tryGetLeveledManifest()
 +    {
 +        for (File dir : sstableDirectories)
 +        {
 +            File manifestFile = new File(dir, cfname + LeveledManifest.EXTENSION);
 +            if (manifestFile.exists())
 +            {
 +                logger.debug("Found manifest at {}", manifestFile);
 +                return manifestFile;
 +            }
 +        }
 +        logger.debug("No level manifest found");
 +        return null;
 +    }
 +
 +    public File getOrCreateLeveledManifest()
 +    {
 +        File manifestFile = tryGetLeveledManifest();
 +        if (manifestFile == null)
 +            manifestFile = new File(sstableDirectories[0], cfname + LeveledManifest.EXTENSION);
 +        return manifestFile;
 +    }
 +
 +    public void snapshotLeveledManifest(String snapshotName) throws IOException
 +    {
 +        File manifest = tryGetLeveledManifest();
 +        if (manifest != null)
 +        {
 +            File snapshotDirectory = getOrCreate(manifest.getParentFile(), SNAPSHOT_SUBDIR,
snapshotName);
 +            CLibrary.createHardLink(manifest, new File(snapshotDirectory, manifest.getName()));
 +        }
 +    }
 +
 +    public boolean snapshotExists(String snapshotName)
 +    {
 +        for (File dir : sstableDirectories)
 +        {
 +            File snapshotDir = new File(dir, join(SNAPSHOT_SUBDIR, snapshotName));
 +            if (snapshotDir.exists())
 +                return true;
 +        }
 +        return false;
 +    }
 +
 +    public void clearSnapshot(String snapshotName) throws IOException
 +    {
 +        // If snapshotName is empty or null, we will delete the entire snapshot directory
 +        String tag = snapshotName == null ? "" : snapshotName;
 +        for (File dir : sstableDirectories)
 +        {
 +            File snapshotDir = new File(dir, join(SNAPSHOT_SUBDIR, tag));
 +            if (snapshotDir.exists())
 +            {
 +                if (logger.isDebugEnabled())
 +                    logger.debug("Removing snapshot directory " + snapshotDir);
 +                FileUtils.deleteRecursive(snapshotDir);
 +            }
 +        }
 +    }
 +
 +    private static File getOrCreate(File base, String... subdirs)
 +    {
 +        File dir = subdirs == null || subdirs.length == 0 ? base : new File(base, join(subdirs));
 +        if (dir.exists())
 +        {
 +            if (!dir.isDirectory())
 +                throw new IOError(new IOException(String.format("Invalid directory path
%s: path exists but is not a directory", dir)));
 +        }
 +        else if (!dir.mkdirs())
 +        {
 +            throw new IOError(new IOException("Unable to create directory " + dir));
 +        }
 +        return dir;
 +    }
 +
 +    private static String join(String... s)
 +    {
 +        return StringUtils.join(s, File.separator);
 +    }
 +
 +    /**
 +     * To check if sstables needs migration, we look at the System directory.
 +     * If it contains a directory for the status cf, we'll attempt a sstable
 +     * migration.
 +     * Note that it is mostly harmless to try a migration uselessly, except
 +     * maybe for some wasted cpu cycles.
 +     */
 +    public static boolean sstablesNeedsMigration()
 +    {
 +        if (StorageService.instance.isClientMode())
 +            return false;
 +
 +        boolean hasSystemKeyspace = false;
 +        for (File location : dataFileLocations)
 +        {
 +            File systemDir = new File(location, Table.SYSTEM_TABLE);
 +            hasSystemKeyspace |= (systemDir.exists() && systemDir.isDirectory());
 +            File statusCFDir = new File(systemDir, SystemTable.STATUS_CF);
 +            if (statusCFDir.exists())
 +                return false;
 +        }
 +        if (!hasSystemKeyspace)
 +            // This is a brand new node.
 +            return false;
 +
 +        // Check whether the migration might create too long a filename
 +        int longestLocation = -1;
 +        try
 +        {
 +            for (File loc : dataFileLocations)
 +                longestLocation = Math.max(longestLocation, loc.getCanonicalPath().length());
 +        }
 +        catch (IOException e)
 +        {
 +            throw new IOError(e);
 +        }
 +
 +        // Check that migration won't error out halfway through from too-long paths.  For
Windows, we need to check
 +        // total path length <= 255 (see http://msdn.microsoft.com/en-us/library/aa365247.aspx
and discussion on CASSANDRA-2749);
 +        // elsewhere, we just need to make sure filename is <= 255.
 +        for (KSMetaData ksm : Schema.instance.getTableDefinitions())
 +        {
 +            String ksname = ksm.name;
 +            for (Map.Entry<String, CFMetaData> entry : ksm.cfMetaData().entrySet())
 +            {
 +                String cfname = entry.getKey();
 +
 +                // max path is roughly (guess-estimate) <location>/ksname/cfname/snapshots/1324314347102-somename/ksname-cfname-tmp-hb-65536-Statistics.db
 +                if (System.getProperty("os.name").startsWith("Windows")
 +                    && longestLocation + (ksname.length() + cfname.length()) * 2
+ 63 > 255)
 +                {
 +                    throw new RuntimeException(String.format("Starting with 1.1, keyspace
names and column family " +
 +                                                             "names must be less than %s
characters long. %s/%s doesn't" +
 +                                                             " respect that restriction.
Please rename your " +
 +                                                             "keyspace/column families to
respect that restriction " +
 +                                                             "before updating.", Schema.NAME_LENGTH,
ksname, cfname));
 +                }
 +
 +                if (ksm.name.length() + cfname.length() + 28 > 255)
 +                {
 +                    throw new RuntimeException("Starting with 1.1, the keyspace name is
included in data filenames.  For "
 +                                               + ksm.name + "/" + cfname + ", this puts
you over the largest possible filename of 255 characters");
 +                }
 +            }
 +        }
 +
 +        return true;
 +    }
 +
 +    /**
 +     * Move sstables from the pre-#2749 layout to their new location/names.
 +     * This involves:
 +     *   - moving each sstable to their CF specific directory
 +     *   - rename the sstable to include the keyspace in the filename
 +     *
 +     * Note that this also move leveled manifests, snapshots and backups.
 +     */
 +    public static void migrateSSTables()
 +    {
 +        logger.info("Upgrade from pre-1.1 version detected: migrating sstables to new directory
layout");
 +
 +        for (File location : dataFileLocations)
 +        {
 +            if (!location.exists() || !location.isDirectory())
 +                continue;
 +
 +            for (File ksDir : location.listFiles())
 +            {
 +                if (!ksDir.isDirectory())
 +                    continue;
 +
 +                for (File file : ksDir.listFiles())
 +                    migrateFile(file, ksDir, null);
 +
 +                migrateSnapshots(ksDir);
 +                migrateBackups(ksDir);
 +            }
 +        }
 +    }
 +
 +    private static void migrateSnapshots(File ksDir)
 +    {
 +        File snapshotDir = new File(ksDir, SNAPSHOT_SUBDIR);
 +        if (!snapshotDir.exists())
 +            return;
 +
 +        for (File snapshot : snapshotDir.listFiles())
 +        {
 +            if (!snapshot.isDirectory())
 +                continue;
 +
 +            for (File f : snapshot.listFiles())
 +                migrateFile(f, ksDir, join(SNAPSHOT_SUBDIR, snapshot.getName()));
 +
 +            if (!snapshot.delete())
 +                logger.info("Old snapsot directory {} not deleted by migraation as it is
not empty", snapshot);
 +        }
 +        if (!snapshotDir.delete())
 +            logger.info("Old directory {} not deleted by migration as it is not empty",
snapshotDir);
 +    }
 +
 +    private static void migrateBackups(File ksDir)
 +    {
 +        File backupDir = new File(ksDir, BACKUPS_SUBDIR);
 +        if (!backupDir.exists())
 +            return;
 +
 +        for (File f : backupDir.listFiles())
 +            migrateFile(f, ksDir, BACKUPS_SUBDIR);
 +
 +        if (!backupDir.delete())
 +            logger.info("Old directory {} not deleted by migration as it is not empty",
backupDir);
 +    }
 +
 +    private static void migrateFile(File file, File ksDir, String additionalPath)
 +    {
 +        try
 +        {
 +            if (file.isDirectory())
 +                return;
 +
 +            String name = file.getName();
 +            boolean isManifest = name.endsWith(LeveledManifest.EXTENSION);
 +            String cfname = isManifest
 +                          ? name.substring(0, name.length() - LeveledManifest.EXTENSION.length())
 +                          : name.substring(0, name.indexOf(Component.separator));
 +
 +            int idx = cfname.indexOf(SECONDARY_INDEX_NAME_SEPARATOR); // idx > 0 =>
secondary index
 +            String dirname = idx > 0 ? cfname.substring(0, idx) : cfname;
 +            File destDir = getOrCreate(ksDir, dirname, additionalPath);
 +
 +            File destFile = new File(destDir, isManifest ? name : ksDir.getName() + Component.separator
+ name);
 +            logger.debug(String.format("[upgrade to 1.1] Moving %s to %s", file, destFile));
 +            FileUtils.renameWithConfirm(file, destFile);
 +        }
 +        catch (IOException e)
 +        {
 +            throw new IOError(e);
 +        }
 +    }
 +
 +    // Hack for tests, don't use otherwise
 +    static void overrideDataDirectoriesForTest(String loc)
 +    {
 +        for (int i = 0; i < dataFileLocations.length; ++i)
 +            dataFileLocations[i] = new File(loc);
 +    }
 +
 +    // Hack for tests, don't use otherwise
 +    static void resetDataDirectoriesAfterTest()
 +    {
 +        String[] locations = DatabaseDescriptor.getAllDataFileLocations();
 +        for (int i = 0; i < locations.length; ++i)
 +            dataFileLocations[i] = new File(locations[i]);
 +    }
 +}

http://git-wip-us.apache.org/repos/asf/cassandra/blob/af1dc0f5/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionTask.java
index b66f20b,2a1b415..6d914e3
--- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
@@@ -18,10 -18,9 +18,9 @@@
  
  package org.apache.cassandra.db.compaction;
  
 +import java.io.File;
  import java.io.IOException;
  import java.util.*;
--import java.util.Map.Entry;
  
  import com.google.common.base.Predicates;
  import com.google.common.collect.Iterators;
@@@ -75,9 -76,9 +74,7 @@@ public class CompactionTask extends Abs
          if (!isCompactionInteresting(toCompact))
              return 0;
  
-         File compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(cfs.getExpectedCompactedFileSize(toCompact),
-                                                                                  ensureFreeSpace());
- 
 -        if (compactionFileLocation == null)
 -            compactionFileLocation = cfs.table.getDataFileLocation(cfs.getExpectedCompactedFileSize(toCompact));
 -
++        File compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(cfs.getExpectedCompactedFileSize(toCompact));
          if (compactionFileLocation == null && partialCompactionsAcceptable())
          {
              // If the compaction file path is null that means we have no space left for
this compaction.
@@@ -88,17 -89,14 +85,15 @@@
                  // Note that we have removed files that are still marked as compacting.
                  // This suboptimal but ok since the caller will unmark all the sstables
at the end.
                  toCompact.remove(cfs.getMaxSizeFile(toCompact));
-                 compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(cfs.getExpectedCompactedFileSize(toCompact),
-                                                                                     ensureFreeSpace());
 -                compactionFileLocation = cfs.table.getDataFileLocation(cfs.getExpectedCompactedFileSize(toCompact));
++                compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(cfs.getExpectedCompactedFileSize(toCompact));
              }
          }
 +
          if (compactionFileLocation == null)
          {
-             logger.warn("insufficient space to compact even the two smallest files, aborting");
+             logger.warn("insufficient space to compact; aborting compaction");
              return 0;
          }
-         assert compactionFileLocation != null;
  
          if (DatabaseDescriptor.isSnapshotBeforeCompaction())
              cfs.snapshotWithoutFlush(System.currentTimeMillis() + "-" + "compact-" + cfs.columnFamily);
@@@ -204,10 -198,10 +199,10 @@@
  
          cfs.replaceCompactedSSTables(toCompact, sstables, compactionType);
          // TODO: this doesn't belong here, it should be part of the reader to load when
the tracker is wired up
--        for (Entry<SSTableReader, Map<DecoratedKey, Long>> ssTableReaderMapEntry
: cachedKeyMap.entrySet())
++        for (Map.Entry<SSTableReader, Map<DecoratedKey, Long>> ssTableReaderMapEntry
: cachedKeyMap.entrySet())
          {
              SSTableReader key = ssTableReaderMapEntry.getKey();
--            for (Entry<DecoratedKey, Long> entry : ssTableReaderMapEntry.getValue().entrySet())
++            for (Map.Entry<DecoratedKey, Long> entry : ssTableReaderMapEntry.getValue().entrySet())
                 key.cacheKey(entry.getKey(), entry.getValue());
          }
  

http://git-wip-us.apache.org/repos/asf/cassandra/blob/af1dc0f5/src/java/org/apache/cassandra/io/sstable/Descriptor.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/af1dc0f5/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
----------------------------------------------------------------------


Mime
View raw message