cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject svn commit: r1179370 - in /cassandra/branches/cassandra-1.0.0: ./ contrib/ interface/thrift/gen-java/org/apache/cassandra/thrift/ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/db/commitlog/ test/unit/org/apache/cassandra/db/
Date Wed, 05 Oct 2011 18:33:39 GMT
Author: jbellis
Date: Wed Oct  5 18:33:38 2011
New Revision: 1179370

URL: http://svn.apache.org/viewvc?rev=1179370&view=rev
Log:
merge from 0.8

Modified:
    cassandra/branches/cassandra-1.0.0/   (props changed)
    cassandra/branches/cassandra-1.0.0/CHANGES.txt
    cassandra/branches/cassandra-1.0.0/contrib/   (props changed)
    cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
  (props changed)
    cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
  (props changed)
    cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
  (props changed)
    cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
  (props changed)
    cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
  (props changed)
    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/Truncation.java
    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
    cassandra/branches/cassandra-1.0.0/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java

Propchange: cassandra/branches/cassandra-1.0.0/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Oct  5 18:33:38 2011
@@ -1,7 +1,7 @@
 /cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7:1026516-1170333,1172024
 /cassandra/branches/cassandra-0.7.0:1053690-1055654
-/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1178554,1178785
+/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1178554,1178785,1179359,1179364
 /cassandra/branches/cassandra-0.8.0:1125021-1130369
 /cassandra/branches/cassandra-0.8.1:1101014-1125018
 /cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689

Modified: cassandra/branches/cassandra-1.0.0/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/CHANGES.txt?rev=1179370&r1=1179369&r2=1179370&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/CHANGES.txt (original)
+++ cassandra/branches/cassandra-1.0.0/CHANGES.txt Wed Oct  5 18:33:38 2011
@@ -17,6 +17,7 @@
  * Fix broken CompressedRandomAccessReaderTest (CASSANDRA-3298)
  * Nodetool no longer leaks threads and closes JMX connections (CASSANDRA-3309)
  * (CQL) fix type information returned for wildcard queries (CASSANDRA-3311)
+ * fix truncate allowing data to be replayed post-restart (CASSANDRA-3297)
 
 
 1.0.0-rc2

Propchange: cassandra/branches/cassandra-1.0.0/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Oct  5 18:33:38 2011
@@ -1,7 +1,7 @@
 /cassandra/branches/cassandra-0.6/contrib:922689-1052356,1052358-1053452,1053454,1053456-1068009
 /cassandra/branches/cassandra-0.7/contrib:1026516-1170333,1172024
 /cassandra/branches/cassandra-0.7.0/contrib:1053690-1055654
-/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1178554,1178785
+/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1178554,1178785,1179359,1179364
 /cassandra/branches/cassandra-0.8.0/contrib:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/contrib:1101014-1125018
 /cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689

Propchange: cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Oct  5 18:33:38 2011
@@ -1,7 +1,7 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1170333,1172024
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1178554,1178785
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1178554,1178785,1179359,1179364
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1101014-1125018
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689

Propchange: cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Oct  5 18:33:38 2011
@@ -1,7 +1,7 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1170333,1172024
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1178554,1178785
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1178554,1178785,1179359,1179364
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1101014-1125018
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689

Propchange: cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Oct  5 18:33:38 2011
@@ -1,7 +1,7 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1170333,1172024
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1178554,1178785
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1178554,1178785,1179359,1179364
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1101014-1125018
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689

Propchange: cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Oct  5 18:33:38 2011
@@ -1,7 +1,7 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1170333,1172024
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1178554,1178785
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1178554,1178785,1179359,1179364
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1101014-1125018
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689

Propchange: cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Oct  5 18:33:38 2011
@@ -1,7 +1,7 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1170333,1172024
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1178554,1178785
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1178554,1178785,1179359,1179364
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1101014-1125018
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689

Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1179370&r1=1179369&r2=1179370&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
+++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
Wed Oct  5 18:33:38 2011
@@ -714,7 +714,7 @@ public class ColumnFamilyStore implement
 
         if (clean)
         {
-            logger.debug("forceFlush requested but everything is clean");
+            logger.debug("forceFlush requested but everything is clean in {}", columnFamily);
             return null;
         }
 
@@ -1593,7 +1593,7 @@ public class ColumnFamilyStore implement
      * @return a Future to the delete operation. Call the future's get() to make
      * sure the column family has been deleted
      */
-    public Future<?> truncate() throws IOException
+    public Future<?> truncate() throws IOException, ExecutionException, InterruptedException
     {
         // We have two goals here:
         // - truncate should delete everything written before truncate was invoked
@@ -1611,11 +1611,15 @@ public class ColumnFamilyStore implement
         //
         // Bonus bonus: simply forceFlush of all the CF is not enough, because if
         // for a given column family the memtable is clean, forceFlush will return
-        // immediately, even though there could be a memtable being flush at the same
-        // time.  So to guarantee that all segments can be cleaned out, we need
+        // immediately, even though there could be a memtable being flushed at the same
+        // time.  So to guarantee that all segments can be cleaned out, we need to
         // "waitForActiveFlushes" after the new segment has been created.
+        logger.debug("truncating {}", columnFamily);
+        // flush the CF being truncated before forcing the new segment
+        forceBlockingFlush();
         CommitLog.instance.forceNewSegment();
         ReplayPosition position = CommitLog.instance.getContext();
+        // now flush everyone else.  re-flushing ourselves is not necessary, but harmless
         for (ColumnFamilyStore cfs : ColumnFamilyStore.all())
             cfs.forceFlush();
         waitForActiveFlushes();

Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/Truncation.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/Truncation.java?rev=1179370&r1=1179369&r2=1179370&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/Truncation.java (original)
+++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/Truncation.java Wed
Oct  5 18:33:38 2011
@@ -55,15 +55,6 @@ public class Truncation implements Messa
         this.columnFamily = columnFamily;
     }
 
-    /**
-     * This is equivalent to calling commit. Applies the changes to
-     * to the table that is obtained by calling Table.open().
-     */
-    public void apply() throws IOException
-    {
-        Table.open(keyspace).getColumnFamilyStore(columnFamily).truncate();
-    }
-
     public Message getMessage(Integer version) throws IOException
     {
         FastByteArrayOutputStream bos = new FastByteArrayOutputStream();

Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLog.java?rev=1179370&r1=1179369&r2=1179370&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
(original)
+++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLog.java
Wed Oct  5 18:33:38 2011
@@ -131,6 +131,8 @@ public class CommitLog implements Commit
 
     public void resetUnsafe()
     {
+        for (CommitLogSegment segment : segments)
+            segment.close();
         segments.clear();
         segments.add(new CommitLogSegment());
     }
@@ -484,7 +486,6 @@ public class CommitLog implements Commit
         if (segment.isSafeToDelete() && iter.hasNext())
         {
             logger.info("Discarding obsolete commit log:" + segment);
-            segment.close();
             FileUtils.deleteAsync(segment.getPath());
             // usually this will be the first (remaining) segment, but not always, if segment
A contains
             // writes to a CF that is unflushed but is followed by segment B whose CFs are
all flushed.
@@ -532,34 +533,26 @@ public class CommitLog implements Commit
         return getSize();
     }
 
-    public void forceNewSegment()
+    public void forceNewSegment() throws ExecutionException, InterruptedException
     {
         Callable<?> task = new Callable()
         {
             public Object call() throws IOException
             {
-                createNewSegment();
+                if (currentSegment().length() > 0)
+                    createNewSegment();
                 return null;
             }
         };
 
-        try
-        {
-            executor.submit(task).get();
-        }
-        catch (InterruptedException e)
-        {
-            throw new AssertionError(e);
-        }
-        catch (ExecutionException e)
-        {
-            throw new RuntimeException(e);
-        }
+        executor.submit(task).get();
     }
 
     private void createNewSegment() throws IOException
     {
+        assert !segments.isEmpty();
         sync();
+        segments.getLast().close();
         segments.add(new CommitLogSegment());
 
         // Maintain desired CL size cap

Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java?rev=1179370&r1=1179369&r2=1179370&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
(original)
+++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/commitlog/CommitLogSegment.java
Wed Oct  5 18:33:38 2011
@@ -48,6 +48,7 @@ public class CommitLogSegment
 
     public final long id;
     private final SequentialWriter logWriter;
+    private long finalSize = -1;
 
     // cache which cf is dirty in this segment to avoid having to lookup all ReplayPositions
to decide if we could delete this segment
     public final Map<Integer, Integer> cfLastWrite = new HashMap<Integer, Integer>();
@@ -159,6 +160,9 @@ public class CommitLogSegment
 
     public long length()
     {
+        if (finalSize >= 0)
+            return finalSize;
+        
         try
         {
             return logWriter.length();
@@ -171,8 +175,12 @@ public class CommitLogSegment
 
     public void close()
     {
+        if (finalSize >= 0)
+            return;
+
         try
         {
+            finalSize = logWriter.length();
             logWriter.close();
         }
         catch (IOException e)

Modified: cassandra/branches/cassandra-1.0.0/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java?rev=1179370&r1=1179369&r2=1179370&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java
(original)
+++ cassandra/branches/cassandra-1.0.0/test/unit/org/apache/cassandra/db/RecoveryManagerTruncateTest.java
Wed Oct  5 18:33:38 2011
@@ -48,7 +48,7 @@ public class RecoveryManagerTruncateTest
 		RowMutation rm;
 		ColumnFamily cf;
 
-		// trucate clears memtable
+		// add a single cell
 		rm = new RowMutation("Keyspace1", ByteBufferUtil.bytes("keymulti"));
 		cf = ColumnFamily.create("Keyspace1", "Standard1");
 		cf.addColumn(column("col1", "val1", 1L));
@@ -60,22 +60,11 @@ public class RecoveryManagerTruncateTest
 
 		// and now truncate it
 		cfs.truncate().get();
+        CommitLog.instance.resetUnsafe();
 		CommitLog.recover();
 
 		// and validate truncation.
 		assertNull(getFromTable(table, "Standard1", "keymulti", "col1"));
-
-		// truncate clears sstable
-		rm = new RowMutation("Keyspace1", ByteBufferUtil.bytes("keymulti"));
-		cf = ColumnFamily.create("Keyspace1", "Standard1");
-		cf.addColumn(column("col1", "val1", 1L));
-		rm.add(cf);
-		rm.apply();
-		cfs.forceBlockingFlush();
-		cfs.truncate().get();
-        CommitLog.instance.resetUnsafe();
-		CommitLog.recover();
-		assertNull(getFromTable(table, "Standard1", "keymulti", "col1"));
 	}
 
 	private IColumn getFromTable(Table table, String cfName, String keyName, String columnName)



Mime
View raw message