cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject svn commit: r776716 - in /incubator/cassandra/trunk: ./ interface/gen-java/org/apache/cassandra/service/ lib/ src/java/org/ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/io/ test/unit/org/ test/unit/org/apache/cassandra/db/
Date Wed, 20 May 2009 14:44:41 GMT
Author: jbellis
Date: Wed May 20 14:44:41 2009
New Revision: 776716

URL: http://svn.apache.org/viewvc?rev=776716&view=rev
Log:
merge from 0.3 branch

Removed:
    incubator/cassandra/trunk/lib/activation.jar
Modified:
    incubator/cassandra/trunk/   (props changed)
    incubator/cassandra/trunk/build.xml
    incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Cassandra.java
  (props changed)
    incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/CqlResult_t.java
  (props changed)
    incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java
  (props changed)
    incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/NotFoundException.java
  (props changed)
    incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/batch_mutation_super_t.java
  (props changed)
    incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/batch_mutation_t.java
  (props changed)
    incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/column_t.java
  (props changed)
    incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/superColumn_t.java
  (props changed)
    incubator/cassandra/trunk/src/java/org/   (props changed)
    incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/db/MinorCompactionManager.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTable.java
    incubator/cassandra/trunk/test/unit/org/   (props changed)
    incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/OneCompactionTest.java

Propchange: incubator/cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1 +1 @@
-/incubator/cassandra/branches/cassandra-0.3:774578-775981
+/incubator/cassandra/branches/cassandra-0.3:774578-776715

Modified: incubator/cassandra/trunk/build.xml
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/build.xml?rev=776716&r1=776715&r2=776716&view=diff
==============================================================================
--- incubator/cassandra/trunk/build.xml (original)
+++ incubator/cassandra/trunk/build.xml Wed May 20 14:44:41 2009
@@ -178,7 +178,7 @@
       </copy> 
       
       <tar compression="gzip" longfile="gnu"
-        destfile="${build.dir}/${final.name}.tgz">
+        destfile="${build.dir}/${final.name}-bin.tar.gz">
 
         <tarfileset dir="${dist.dir}"
           prefix="${final.name}">
@@ -191,6 +191,15 @@
           <include name="bin/cassandra"/>
         </tarfileset>	
       </tar>
+      <tar compression="gzip" longfile="gnu"
+        destfile="${build.dir}/${final.name}-src.tar.gz">
+
+        <tarfileset dir="${basedir}"
+          prefix="${final.name}-src">
+          <include name="**"/>
+          <exclude name="build/**" />
+        </tarfileset>
+      </tar>
     </target>
 
   <target name="build-test" depends="build" description="Build the Cassandra classes">

Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-776715
 /incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Cassandra.java:749219-768588

Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/CqlResult_t.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/CqlResult_t.java:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/CqlResult_t.java:774578-776715
 /incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/CqlResult_t.java:749219-768588

Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-776715
 /incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:749219-768588

Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-776715
 /incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:749219-768588

Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/batch_mutation_super_t.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/batch_mutation_super_t.java:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/batch_mutation_super_t.java:774578-776715
 /incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/batch_mutation_super_t.java:749219-768588

Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/batch_mutation_t.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/batch_mutation_t.java:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/batch_mutation_t.java:774578-776715
 /incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/batch_mutation_t.java:749219-768588

Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/column_t.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-776715
 /incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/column_t.java:749219-768588

Propchange: incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/superColumn_t.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-776715
 /incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:749219-768588

Propchange: incubator/cassandra/trunk/src/java/org/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/src/java/org:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/src/java/org:774578-776715
 /incubator/cassandra/trunk/src/java/org:749219-769885

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=776716&r1=776715&r2=776716&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Wed
May 20 14:44:41 2009
@@ -56,7 +56,6 @@
 {
     private static Logger logger_ = Logger.getLogger(ColumnFamilyStore.class);
 
-    private static int COMPACTION_THRESHOLD = 4; // compact this many sstables at a time
     private static final int BUFSIZE = 128 * 1024 * 1024;
     private static final int COMPACTION_MEMORY_THRESHOLD = 1 << 30;
 
@@ -770,8 +769,8 @@
             lock_.writeLock().unlock();
         }
 
-        if ((ssTableSize >= COMPACTION_THRESHOLD && !isCompacting_.get())
-            || (isCompacting_.get() && ssTableSize % COMPACTION_THRESHOLD == 0))
+        if ((ssTableSize >= MinorCompactionManager.COMPACTION_THRESHOLD && !isCompacting_.get())
+            || (isCompacting_.get() && ssTableSize % MinorCompactionManager.COMPACTION_THRESHOLD
== 0))
         {
             logger_.debug("Submitting for  compaction ...");
             MinorCompactionManager.instance().submit(ColumnFamilyStore.this);
@@ -860,11 +859,6 @@
         return buckets.keySet();
     }
 
-    public int doCompaction() throws IOException
-    {
-        return doCompaction(COMPACTION_THRESHOLD);
-    }
-
     /*
      * Break the files into buckets and then compact.
      */
@@ -1294,6 +1288,7 @@
     */
     private int doFileCompaction(List<String> files, int minBufferSize) throws IOException
     {
+        logger_.info("Compacting [" + StringUtils.join(files, ",") + "]");
         String compactionFileLocation = DatabaseDescriptor.getCompactionFileLocation(getExpectedCompactedFileSize(files));
         // If the compaction file path is null that means we have no space left for this
compaction.
         // try again w/o the largest one.
@@ -1314,6 +1309,7 @@
 
         if (pq.isEmpty())
         {
+            logger_.warn("Nothing to compact (all files empty or corrupt)");
             // TODO clean out bad files, if any
             return 0;
         }

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/MinorCompactionManager.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/MinorCompactionManager.java?rev=776716&r1=776715&r2=776716&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/MinorCompactionManager.java
(original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/MinorCompactionManager.java
Wed May 20 14:44:41 2009
@@ -46,7 +46,8 @@
     private static MinorCompactionManager instance_;
     private static Lock lock_ = new ReentrantLock();
     private static Logger logger_ = Logger.getLogger(MinorCompactionManager.class);
-    final static long intervalInMins_ = 5;
+    private static final long intervalInMins_ = 5;
+    static final int COMPACTION_THRESHOLD = 4; // compact this many sstables at a time
 
     public static MinorCompactionManager instance()
     {
@@ -66,33 +67,6 @@
         return instance_;
     }
 
-    class FileCompactor implements Callable<Integer>
-    {
-        private ColumnFamilyStore columnFamilyStore_;
-
-        FileCompactor(ColumnFamilyStore columnFamilyStore)
-        {
-            columnFamilyStore_ = columnFamilyStore;
-        }
-
-        public Integer call()
-        {
-            logger_.debug("Started compaction ..." + columnFamilyStore_.columnFamily_);
-            try
-            {
-                return columnFamilyStore_.doCompaction();
-            }
-            catch (IOException e)
-            {
-                throw new RuntimeException(e);
-            }
-            finally
-            {
-                logger_.debug("Finished compaction ..." + columnFamilyStore_.columnFamily_);
-            }
-        }
-    }
-
     class FileCompactor2 implements Callable<Boolean>
     {
         private ColumnFamilyStore columnFamilyStore_;
@@ -138,9 +112,9 @@
 
         public void run()
         {
-            logger_.debug("Started  Major compaction ..."+columnFamilyStore_.columnFamily_);
+            logger_.debug("Started  Major compaction for " + columnFamilyStore_.columnFamily_);
             columnFamilyStore_.doMajorCompaction(skip_);
-            logger_.debug("Finished Major compaction ..."+columnFamilyStore_.columnFamily_);
+            logger_.debug("Finished Major compaction for " + columnFamilyStore_.columnFamily_);
         }
     }
 
@@ -183,22 +157,41 @@
 
     public void submitPeriodicCompaction(final ColumnFamilyStore columnFamilyStore)
     {
-        Runnable runnable = new Runnable() // having to wrap Callable in Runnable is retarded
but that's what the API insists on.
+        Runnable runnable = new Runnable()
         {
             public void run()
             {
-                new FileCompactor(columnFamilyStore).call();
+                try
+                {
+                    columnFamilyStore.doCompaction(COMPACTION_THRESHOLD);
+                }
+                catch (IOException e)
+                {
+                    throw new RuntimeException(e);
+                }
             }
         };
     	compactor_.scheduleWithFixedDelay(runnable, MinorCompactionManager.intervalInMins_,
     			MinorCompactionManager.intervalInMins_, TimeUnit.MINUTES);       
     }
 
-    public Future<Integer> submit(ColumnFamilyStore columnFamilyStore)
+    public Future<Integer> submit(final ColumnFamilyStore columnFamilyStore)
     {
-        return compactor_.submit(new FileCompactor(columnFamilyStore));
+        return submit(columnFamilyStore, COMPACTION_THRESHOLD);
     }
-    
+
+    Future<Integer> submit(final ColumnFamilyStore columnFamilyStore, final int threshold)
+    {
+        Callable<Integer> callable = new Callable<Integer>()
+        {
+            public Integer call() throws IOException
+            {
+                return columnFamilyStore.doCompaction(threshold);
+            }
+        };
+        return compactor_.submit(callable);
+    }
+
     public void submitCleanup(ColumnFamilyStore columnFamilyStore)
     {
         compactor_.submit(new CleanupCompactor(columnFamilyStore));

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTable.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTable.java?rev=776716&r1=776715&r2=776716&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTable.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTable.java Wed May 20 14:44:41
2009
@@ -220,7 +220,7 @@
         }
 
         File file = new File(dataFile);
-        assert file.exists();
+        assert file.exists() : "attempted to delete non-existing file " + dataFile;
         /* delete the data file */
         if (!file.delete())
         {

Propchange: incubator/cassandra/trunk/test/unit/org/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 20 14:44:41 2009
@@ -1,2 +1,2 @@
-/incubator/cassandra/branches/cassandra-0.3/test/unit/org:774578-775981
+/incubator/cassandra/branches/cassandra-0.3/test/unit/org:774578-776715
 /incubator/cassandra/trunk/test/unit/org:749219-768583

Modified: incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/OneCompactionTest.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/OneCompactionTest.java?rev=776716&r1=776715&r2=776716&view=diff
==============================================================================
--- incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/OneCompactionTest.java (original)
+++ incubator/cassandra/trunk/test/unit/org/apache/cassandra/db/OneCompactionTest.java Wed
May 20 14:44:41 2009
@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
 import java.util.Set;
 import java.util.HashSet;
 
@@ -44,7 +45,9 @@
             store.forceBlockingFlush();
             assertEquals(table.getKeyRange("", "", 10000).size(), inserted.size());
         }
-        store.doCompaction(2);
+        Future<Integer> ft = MinorCompactionManager.instance().submit(store, 2);
+        ft.get();
+        assertEquals(1, store.getSSTableFilenames().size());
         assertEquals(table.getKeyRange("", "", 10000).size(), inserted.size());
     }
 
@@ -57,6 +60,6 @@
     @Test
     public void testCompaction2() throws IOException, ExecutionException, InterruptedException
     {
-        testCompaction("Standard2", 500);
+        testCompaction("Standard2", 5);
     }
 }



Mime
View raw message