cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From slebre...@apache.org
Subject svn commit: r1205319 - in /cassandra/trunk: ./ contrib/ interface/thrift/gen-java/org/apache/cassandra/thrift/ src/java/org/apache/cassandra/db/context/ test/unit/org/apache/cassandra/db/context/
Date Wed, 23 Nov 2011 08:04:46 GMT
Author: slebresne
Date: Wed Nov 23 08:04:45 2011
New Revision: 1205319

URL: http://svn.apache.org/viewvc?rev=1205319&view=rev
Log:
merge from 1.0

Modified:
    cassandra/trunk/   (props changed)
    cassandra/trunk/CHANGES.txt
    cassandra/trunk/contrib/   (props changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java 
 (props changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java   (props
changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
  (props changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
  (props changed)
    cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
  (props changed)
    cassandra/trunk/src/java/org/apache/cassandra/db/context/CounterContext.java
    cassandra/trunk/test/unit/org/apache/cassandra/db/context/CounterContextTest.java

Propchange: cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 08:04:45 2011
@@ -1,10 +1,10 @@
 /cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7:1026516-1183000
 /cassandra/branches/cassandra-0.7.0:1053690-1055654
-/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884
+/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884,1205316
 /cassandra/branches/cassandra-0.8.0:1125021-1130369
 /cassandra/branches/cassandra-0.8.1:1101014-1125018
-/cassandra/branches/cassandra-1.0:1167085-1204646,1204869,1204893
+/cassandra/branches/cassandra-1.0:1167085-1204646,1204869,1204893,1205317
 /cassandra/branches/cassandra-1.0.0:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
 /cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689
 /cassandra/tags/cassandra-0.8.0-rc1:1102511-1125020

Modified: cassandra/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1205319&r1=1205318&r2=1205319&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Wed Nov 23 08:04:45 2011
@@ -21,6 +21,7 @@
  * fix incorrect query results due to invalid max timestamp (CASSANDRA-3510)
 Merged from 0.8:
  * fix concurrence issue in the FailureDetector (CASSANDRA-3519)
+ * fix array out of bounds error in counter shard removal (CASSANDRA-3514)
 
 1.0.3
  * revert name-based query defragmentation aka CASSANDRA-2503 (CASSANDRA-3491)

Propchange: cassandra/trunk/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 08:04:45 2011
@@ -1,10 +1,10 @@
 /cassandra/branches/cassandra-0.6/contrib:922689-1052356,1052358-1053452,1053454,1053456-1068009
 /cassandra/branches/cassandra-0.7/contrib:1026516-1183000
 /cassandra/branches/cassandra-0.7.0/contrib:1053690-1055654
-/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884
+/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884,1205316
 /cassandra/branches/cassandra-0.8.0/contrib:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/contrib:1101014-1125018
-/cassandra/branches/cassandra-1.0/contrib:1167085-1204646,1204869,1204893
+/cassandra/branches/cassandra-1.0/contrib:1167085-1204646,1204869,1204893,1205317
 /cassandra/branches/cassandra-1.0.0/contrib:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
 /cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689
 /cassandra/tags/cassandra-0.8.0-rc1/contrib:1102511-1125020

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 08:04:45 2011
@@ -1,10 +1,10 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1183000
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884,1205316
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167085-1204646,1204869,1204893
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167085-1204646,1204869,1204893,1205317
 /cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689
 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1102511-1125020

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 08:04:45 2011
@@ -1,10 +1,10 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1183000
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884,1205316
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167085-1204646,1204869,1204893
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167085-1204646,1204869,1204893,1205317
 /cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689
 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1102511-1125020

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 08:04:45 2011
@@ -1,10 +1,10 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1183000
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884,1205316
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167085-1204646,1204869,1204893
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167085-1204646,1204869,1204893,1205317
 /cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689
 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1102511-1125020

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 08:04:45 2011
@@ -1,10 +1,10 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1183000
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884,1205316
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167085-1204646,1204869,1204893
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167085-1204646,1204869,1204893,1205317
 /cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689
 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1102511-1125020

Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 08:04:45 2011
@@ -1,10 +1,10 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1183000
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284,1200226,1200471,1204884,1205316
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1101014-1125018
-/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167085-1204646,1204869,1204893
+/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167085-1204646,1204869,1204893,1205317
 /cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243
 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689
 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1102511-1125020

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/context/CounterContext.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/context/CounterContext.java?rev=1205319&r1=1205318&r2=1205319&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/context/CounterContext.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/context/CounterContext.java Wed Nov 23
08:04:45 2011
@@ -657,6 +657,7 @@ public class CounterContext implements I
         int hlength = headerLength(context);
         ContextState state = new ContextState(context, hlength);
         int removedShards = 0;
+        int removedDelta = 0;
         while (state.hasRemaining())
         {
             long clock = state.getClock();
@@ -684,7 +685,11 @@ public class CounterContext implements I
                 }
 
                 if (-((int)(clock / 1000)) < gcBefore)
+                {
                     removedShards++;
+                    if (state.isDelta())
+                        removedDelta++;
+                }
             }
             state.moveToNext();
         }
@@ -692,9 +697,9 @@ public class CounterContext implements I
         if (removedShards == 0)
             return context;
 
-
-        int removedHeaderSize = removedShards * HEADER_ELT_LENGTH;
-        int newSize = context.remaining() - removedHeaderSize - (removedShards * STEP_LENGTH);
+        int removedHeaderSize = removedDelta * HEADER_ELT_LENGTH;
+        int removedBodySize = removedShards * STEP_LENGTH;
+        int newSize = context.remaining() - removedHeaderSize - removedBodySize;
         int newHlength = hlength - removedHeaderSize;
         ByteBuffer cleanedContext = HeapAllocator.instance.allocate(newSize);
         cleanedContext.putShort(cleanedContext.position(), (short) ((newHlength - HEADER_SIZE_LENGTH)
/ HEADER_ELT_LENGTH));
@@ -704,10 +709,11 @@ public class CounterContext implements I
         while (state.hasRemaining())
         {
             long clock = state.getClock();
-            if (!(clock < 0 && state.getCount() == 0))
+            if (clock >= 0 || state.getCount() != 0 || -((int)(clock / 1000)) >= gcBefore)
             {
                 state.copyTo(cleaned);
             }
+
             state.moveToNext();
         }
         return cleanedContext;

Modified: cassandra/trunk/test/unit/org/apache/cassandra/db/context/CounterContextTest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/db/context/CounterContextTest.java?rev=1205319&r1=1205318&r2=1205319&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/db/context/CounterContextTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/db/context/CounterContextTest.java Wed
Nov 23 08:04:45 2011
@@ -423,4 +423,81 @@ public class CounterContextTest
         assert cc.total(ctx.context) == cc.total(cleaned);
         assert cleaned.remaining() == ctx.context.remaining() - stepLength - 2;
     }
+
+    @Test
+    public void testRemoveOldShardsNotAllExpiring()
+    {
+        runRemoveOldShardsNotAllExpiring(HeapAllocator.instance);
+        runRemoveOldShardsNotAllExpiring(bumpedSlab());
+    }
+
+    private void runRemoveOldShardsNotAllExpiring(Allocator allocator)
+    {
+        NodeId id1 = NodeId.fromInt(1);
+        NodeId id3 = NodeId.fromInt(3);
+        NodeId id6 = NodeId.fromInt(6);
+        List<NodeId.NodeIdRecord> records = new ArrayList<NodeId.NodeIdRecord>();
+        records.add(new NodeId.NodeIdRecord(id1, 2L));
+        records.add(new NodeId.NodeIdRecord(id3, 4L));
+        records.add(new NodeId.NodeIdRecord(id6, 10L));
+
+        ContextState ctx = ContextState.allocate(6, 3, allocator);
+        ctx.writeElement(id1, 0L, 1L, true);
+        ctx.writeElement(NodeId.fromInt(2), 0L, 2L);
+        ctx.writeElement(id3, 0L, 3L, true);
+        ctx.writeElement(NodeId.fromInt(4), 0L, 3L);
+        ctx.writeElement(NodeId.fromInt(5), 0L, 3L, true);
+        ctx.writeElement(id6, 0L, 6L);
+
+        int timeFirstMerge = (int)(System.currentTimeMillis() / 1000);
+
+        // First, only merge the first id
+        ByteBuffer merger = cc.computeOldShardMerger(ctx.context, records, 3L);
+        ByteBuffer merged = cc.merge(ctx.context, merger, allocator);
+        assert cc.total(ctx.context) == cc.total(merged);
+
+        try
+        {
+            Thread.sleep(2000);
+        }
+        catch (InterruptedException e)
+        {
+            throw new AssertionError();
+        }
+
+        // merge the second one
+        ByteBuffer merger2 = cc.computeOldShardMerger(merged, records, 7L);
+        ByteBuffer merged2 = cc.merge(merged, merger2, allocator);
+        assert cc.total(ctx.context) == cc.total(merged2);
+
+        ByteBuffer cleaned = cc.removeOldShards(merged2, timeFirstMerge + 1);
+        assert cc.total(ctx.context) == cc.total(cleaned);
+        assert cleaned.remaining() == ctx.context.remaining();
+
+        // We should have cleaned id1 but not id3
+        ContextState m = new ContextState(cleaned);
+        m.moveToNext();
+        assert m.getNodeId().equals(id3);
+
+    }
+
+    @Test
+    public void testRemoveNotDeltaOldShards()
+    {
+        runRemoveNotDeltaOldShards(HeapAllocator.instance);
+        runRemoveNotDeltaOldShards(bumpedSlab());
+    }
+
+    private void runRemoveNotDeltaOldShards(Allocator allocator)
+    {
+        ContextState ctx = ContextState.allocate(4, 1, allocator);
+        ctx.writeElement(NodeId.fromInt(1), 1L, 1L, true);
+        ctx.writeElement(NodeId.fromInt(2), -System.currentTimeMillis(), 0L);
+        ctx.writeElement(NodeId.fromInt(3), -System.currentTimeMillis(), 0L);
+        ctx.writeElement(NodeId.fromInt(4), -System.currentTimeMillis(), 0L);
+
+        ByteBuffer cleaned = cc.removeOldShards(ctx.context, (int)(System.currentTimeMillis()
/ 1000) + 1);
+        assert cc.total(ctx.context) == cc.total(cleaned);
+        assert cleaned.remaining() == ctx.context.remaining() - 3 * stepLength;
+    }
 }



Mime
View raw message