cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject svn commit: r906629 - in /incubator/cassandra/trunk/src/java/org/apache/cassandra: db/ io/ service/
Date Thu, 04 Feb 2010 20:01:17 GMT
Author: jbellis
Date: Thu Feb  4 20:01:16 2010
New Revision: 906629

URL: http://svn.apache.org/viewvc?rev=906629&view=rev
Log:
replace gc after each compaction w/ gc before compaction/flush only if we need it for the
file space
patch by jbellis; tested by Brandon Williams for CASSANDRA-724

Modified:
    incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableDeletingReference.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableReader.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java?rev=906629&r1=906628&r2=906629&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/BinaryMemtable.java Thu Feb
 4 20:01:16 2010
@@ -120,7 +120,7 @@
     public SSTableReader writeSortedContents(List<DecoratedKey> sortedKeys) throws
IOException
     {
         logger.info("Writing " + this);
-        String path = cfs.getTempSSTablePath();
+        String path = cfs.getFlushPath();
         SSTableWriter writer = new SSTableWriter(path, sortedKeys.size(), StorageService.getPartitioner());
 
         for (DecoratedKey key : sortedKeys)

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=906629&r1=906628&r2=906629&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Thu
Feb  4 20:01:16 2010
@@ -335,10 +335,13 @@
      * When the sstable object is closed, it will be renamed to a non-temporary
      * format, so incomplete sstables can be recognized and removed on startup.
      */
-    synchronized String getTempSSTablePath()
+    public String getFlushPath()
     {
-        String fname = getTempSSTableFileName();
-        return new File(DatabaseDescriptor.getNextAvailableDataLocation() + File.separator
+ table_, fname).getAbsolutePath();
+        long guessedSize = 2 * DatabaseDescriptor.getMemtableThroughput() * 1024*1024; //
2* adds room for keys, column indexes
+        String location = DatabaseDescriptor.getDataFileLocationForTable(table_, guessedSize);
+        if (location == null)
+            throw new RuntimeException("Insufficient disk space to flush");
+        return new File(location, getTempSSTableFileName()).getAbsolutePath();
     }
 
     public String getTempSSTableFileName()

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java?rev=906629&r1=906628&r2=906629&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CompactionManager.java Thu
Feb  4 20:01:16 2010
@@ -51,8 +51,6 @@
     private static final Logger logger = Logger.getLogger(CompactionManager.class);
     public static final CompactionManager instance;
 
-    private static volatile boolean gcRequested;
-
     private int minimumCompactionThreshold = 4; // compact this many sstables min at a time
     private int maximumCompactionThreshold = 32; // compact this many sstables max at a time
 
@@ -68,36 +66,6 @@
         {
             throw new RuntimeException(e);
         }
-
-        /**
-         * thread that requests GCs to clean out obsolete sstables, sleeping rpc timeout
first so that most in-progress ops can complete
-         * (thus, no longer reference the sstables in question)
-         */
-        new Thread(new Runnable()
-        {
-            final long gcDelay = DatabaseDescriptor.getRpcTimeout();
-
-            public void run()
-            {
-                while (true)
-                {
-                    try
-                    {
-                        Thread.sleep(gcDelay * 10);
-                        if (gcRequested)
-                        {
-                            Thread.sleep(gcDelay);
-                            System.gc();
-                            gcRequested = false;
-                        }
-                    }
-                    catch (InterruptedException e)
-                    {
-                        throw new AssertionError(e);
-                    }
-                }
-            }
-        }, "COMPACTION-GC-INVOKER").start();
     }
 
     private CompactionExecutor executor = new CompactionExecutor();
@@ -337,7 +305,6 @@
 
         SSTableReader ssTable = writer.closeAndOpenReader(DatabaseDescriptor.getKeysCachedFraction(table.name,
cfs.getColumnFamilyName()));
         cfs.replaceCompactedSSTables(sstables, Arrays.asList(ssTable));
-        gcRequested = true;
         submitMinorIfNeeded(cfs);
 
         String format = "Compacted to %s.  %d/%d bytes for %d keys.  Time: %dms.";
@@ -364,7 +331,7 @@
         logger.info("AntiCompacting [" + StringUtils.join(sstables, ",") + "]");
         // Calculate the expected compacted filesize
         long expectedRangeFileSize = cfs.getExpectedCompactedFileSize(sstables) / 2;
-        String compactionFileLocation = DatabaseDescriptor.getDataFileLocationForTable(table.name,
expectedRangeFileSize);
+        String compactionFileLocation = table.getDataFileLocation(expectedRangeFileSize);
         if (compactionFileLocation == null)
         {
             throw new UnsupportedOperationException("disk full");
@@ -438,7 +405,6 @@
         {
             cfs.replaceCompactedSSTables(originalSSTables, sstables);
         }
-        gcRequested = true;
     }
 
     /**

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java?rev=906629&r1=906628&r2=906629&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java Thu Feb  4 20:01:16
2010
@@ -154,7 +154,7 @@
     {
         logger.info("Writing " + this);
         ColumnFamilyStore cfStore = Table.open(table).getColumnFamilyStore(columnfamilyName);
-        SSTableWriter writer = new SSTableWriter(cfStore.getTempSSTablePath(), columnFamilies.size(),
StorageService.getPartitioner());
+        SSTableWriter writer = new SSTableWriter(cfStore.getFlushPath(), columnFamilies.size(),
StorageService.getPartitioner());
 
         DataOutputBuffer buffer = new DataOutputBuffer();
         for (DecoratedKey key : sortedKeys)

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java?rev=906629&r1=906628&r2=906629&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java Thu Feb  4 20:01:16
2010
@@ -21,9 +21,6 @@
 import java.util.*;
 import java.io.IOException;
 import java.io.File;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.concurrent.Future;
 
@@ -32,11 +29,13 @@
 import com.google.common.collect.Iterables;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.io.SSTableDeletingReference;
 import org.apache.cassandra.io.SSTableReader;
 import org.apache.cassandra.io.util.FileUtils;
 
 import java.net.InetAddress;
 
+import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.*;
 import org.apache.cassandra.db.filter.*;
 import org.cliffc.high_scale_lib.NonBlockingHashMap;
@@ -476,7 +475,22 @@
 
     public String getDataFileLocation(long expectedCompactedFileSize)
     {
-        return DatabaseDescriptor.getDataFileLocationForTable(name, expectedCompactedFileSize);
+        String path = DatabaseDescriptor.getDataFileLocationForTable(name, expectedCompactedFileSize);
+        if (path == null)
+        {
+            // retry after GCing to force unmap of compacted SSTables so they can be deleted
+            StorageService.requestGC();
+            try
+            {
+                Thread.sleep(SSTableDeletingReference.RETRY_DELAY * 2);
+            }
+            catch (InterruptedException e)
+            {
+                throw new AssertionError(e);
+            }
+            path = DatabaseDescriptor.getDataFileLocationForTable(name, expectedCompactedFileSize);
+        }
+        return path;
     }
 
     public static String getSnapshotPath(String dataDirPath, String tableName, String snapshotName)

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableDeletingReference.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableDeletingReference.java?rev=906629&r1=906628&r2=906629&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableDeletingReference.java
(original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableDeletingReference.java
Thu Feb  4 20:01:16 2010
@@ -9,12 +9,12 @@
 
 import org.apache.log4j.Logger;
 
-class SSTableDeletingReference extends PhantomReference<SSTableReader>
+public class SSTableDeletingReference extends PhantomReference<SSTableReader>
 {
     private static final Logger logger = Logger.getLogger(SSTableDeletingReference.class);
 
     private static final Timer timer = new Timer("SSTABLE-CLEANUP-TIMER");
-    private static final int RETRY_DELAY = 10000;
+    public static final int RETRY_DELAY = 10000;
 
     public final String path;
     private boolean deleteOnCleanup;
@@ -54,7 +54,7 @@
             {
                 if (attempts++ < DeletionService.MAX_RETRIES)
                 {
-                    timer.schedule(this, 10000);
+                    timer.schedule(this, RETRY_DELAY);
                     return;
                 }
                 else

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableReader.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableReader.java?rev=906629&r1=906628&r2=906629&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableReader.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableReader.java Thu Feb
 4 20:01:16 2010
@@ -564,4 +564,3 @@
         return "FileSSTableMap {" + StringUtils.join(map.keySet(), ", ") + "}";
     }
 }
-

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java?rev=906629&r1=906628&r2=906629&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java Thu
Feb  4 20:01:16 2010
@@ -1472,6 +1472,12 @@
         return isClientMode;
     }
 
+    public static void requestGC()
+    {
+        logger_.info("requesting GC to free disk space");
+        System.gc();
+    }
+
     // Never ever do this at home. Used by tests.
     Map<String, AbstractReplicationStrategy> setReplicationStrategyUnsafe(Map<String,
AbstractReplicationStrategy> replacement)
     {
@@ -1494,5 +1500,4 @@
         tokenMetadata_ = tmd;
         return old;
     }
-
 }



Mime
View raw message