cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marc...@apache.org
Subject [3/4] git commit: Merge branch 'cassandra-2.0' into cassandra-2.1.0
Date Wed, 27 Aug 2014 11:12:14 GMT
Merge branch 'cassandra-2.0' into cassandra-2.1.0

Conflicts:
	CHANGES.txt


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/049762b3
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/049762b3
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/049762b3

Branch: refs/heads/cassandra-2.1
Commit: 049762b3e49ec54cadc0a353520cccc487ea6c2d
Parents: cb4bc27 29befa1
Author: Marcus Eriksson <marcuse@apache.org>
Authored: Wed Aug 27 13:06:54 2014 +0200
Committer: Marcus Eriksson <marcuse@apache.org>
Committed: Wed Aug 27 13:06:54 2014 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 ++
 .../org/apache/cassandra/db/ColumnIndex.java    |  2 +-
 .../org/apache/cassandra/db/RangeTombstone.java |  8 +++++-
 .../db/compaction/LazilyCompactedRow.java       |  3 ++-
 .../apache/cassandra/db/RangeTombstoneTest.java | 26 ++++++++++++++++++++
 5 files changed, 38 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/049762b3/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 32c46bf,abdd561..9c75426
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -16,37 -12,24 +16,39 @@@ Merged from 2.0
   * Configure system.paxos with LeveledCompactionStrategy (CASSANDRA-7753)
   * Fix ALTER clustering column type from DateType to TimestampType when
     using DESC clustering order (CASSANRDA-7797)
 - * Stop inheriting liveRatio and liveRatioComputedAt from previous
 -   memtables (CASSANDRA-7796)
   * Throw EOFException if we run out of chunks in compressed datafile
     (CASSANDRA-7664)
 - * Throw InvalidRequestException when queries contain relations on entire
 -   collection columns (CASSANDRA-7506)
   * Fix PRSI handling of CQL3 row markers for row cleanup (CASSANDRA-7787)
 - * (cqlsh) enable CTRL-R history search with libedit (CASSANDRA-7577)
   * Fix dropping collection when it's the last regular column (CASSANDRA-7744)
   * Properly reject operations on list index with conditions (CASSANDRA-7499)
 - * (Hadoop) allow ACFRW to limit nodes to local DC (CASSANDRA-7252)
 + * Make StreamReceiveTask thread safe and gc friendly (CASSANDRA-7795)
 + * Validate empty cell names from counter updates (CASSANDRA-7798)
++Merged from 1.2:
++ * Track expired tombstones (CASSANDRA-7810)
 +
 +
 +2.1.0-rc6
 + * Fix OOM issue from netty caching over time (CASSANDRA-7743)
 + * json2sstable couldn't import JSON for CQL table (CASSANDRA-7477)
 + * Invalidate all caches on table drop (CASSANDRA-7561)
 + * Skip strict endpoint selection for ranges if RF == nodes (CASSANRA-7765)
 + * Fix Thrift range filtering without 2ary index lookups (CASSANDRA-7741)
 + * Add tracing entries about concurrent range requests (CASSANDRA-7599)
 + * (cqlsh) Fix DESCRIBE for NTS keyspaces (CASSANDRA-7729)
 + * Remove netty buffer ref-counting (CASSANDRA-7735)
 + * Pass mutated cf to index updater for use by PRSI (CASSANDRA-7742)
 + * Include stress yaml example in release and deb (CASSANDRA-7717)
 + * workaround for netty issue causing corrupted data off the wire (CASSANDRA-7695)
 + * cqlsh DESC CLUSTER fails retrieving ring information (CASSANDRA-7687)
 + * Fix binding null values inside UDT (CASSANDRA-7685)
 + * Fix UDT field selection with empty fields (CASSANDRA-7670)
 + * Bogus deserialization of static cells from sstable (CASSANDRA-7684)
 + * Fix NPE on compaction leftover cleanup for dropped table (CASSANDRA-7770)
 +Merged from 2.0:
   * (cqlsh) Wait up to 10 sec for a tracing session (CASSANDRA-7222)
   * Fix NPE in FileCacheService.sizeInBytes (CASSANDRA-7756)
 - * (cqlsh) cqlsh should automatically disable tracing when selecting
 -   from system_traces (CASSANDRA-7641)
 - * (Hadoop) Add CqlOutputFormat (CASSANDRA-6927)
 - * Don't depend on cassandra config for nodetool ring (CASSANDRA-7508)
 - * (cqlsh) Fix failing cqlsh formatting tests (CASSANDRA-7703)
 + * Remove duplicates from StorageService.getJoiningNodes (CASSANDRA-7478)
 + * Clone token map outside of hot gossip loops (CASSANDRA-7758)
   * Fix MS expiring map timeout for Paxos messages (CASSANDRA-7752)
   * Do not flush on truncate if durable_writes is false (CASSANDRA-7750)
   * Give CRR a default input_cql Statement (CASSANDRA-7226)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/049762b3/src/java/org/apache/cassandra/db/ColumnIndex.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/049762b3/src/java/org/apache/cassandra/db/RangeTombstone.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/RangeTombstone.java
index ffae7ba,16fc27a..3f3d675
--- a/src/java/org/apache/cassandra/db/RangeTombstone.java
+++ b/src/java/org/apache/cassandra/db/RangeTombstone.java
@@@ -110,9 -125,10 +110,10 @@@ public class RangeTombstone extends Int
                  return comparator.compare(t1.max, t2.max);
              }
          });
+         public final Set<RangeTombstone> expired = new HashSet<RangeTombstone>();
          private int atomCount;
  
 -        public Tracker(Comparator<ByteBuffer> comparator)
 +        public Tracker(Comparator<Composite> comparator)
          {
              this.comparator = comparator;
          }
@@@ -174,10 -193,10 +178,10 @@@
  
          /**
           * Update this tracker given an {@code atom}.
 -         * If column is a Column, check if any tracked range is useless and
 +         * If column is a Cell, check if any tracked range is useless and
           * can be removed. If it is a RangeTombstone, add it to this tracker.
           */
-         public void update(OnDiskAtom atom)
+         public void update(OnDiskAtom atom, boolean isExpired)
          {
              if (atom instanceof RangeTombstone)
              {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/049762b3/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java
index 1f8c7dc,9573874..0ae41ee
--- a/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java
+++ b/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java
@@@ -238,8 -246,9 +238,9 @@@ public class LazilyCompactedRow extend
                  RangeTombstone t = tombstone;
                  tombstone = null;
  
-                 if (t.data.isGcAble(controller.gcBefore))
 -                if (shouldPurge && t.data.isGcAble(controller.gcBefore))
++                if (t.timestamp() < maxPurgeableTimestamp && t.data.isGcAble(controller.gcBefore))
                  {
+                     indexBuilder.tombstoneTracker().update(t, true);
                      return null;
                  }
                  else

http://git-wip-us.apache.org/repos/asf/cassandra/blob/049762b3/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
index ecf06d3,7f1238f..45fd61f
--- a/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
+++ b/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
@@@ -120,112 -105,38 +120,138 @@@ public class RangeTombstoneTest extend
          cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(7), b(30),
false, Integer.MAX_VALUE, System.currentTimeMillis()));
  
          for (int i : new int[]{ 7, 8, 9, 11, 13, 15, 17, 28, 29, 30 })
 -            assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live";
 +            assert isLive(cf, cf.getColumn(b(i))) : "Cell " + i + " should be live";
          for (int i : new int[]{ 10, 12, 14, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 })
 -            assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live";
 +            assert !isLive(cf, cf.getColumn(b(i))) : "Cell " + i + " shouldn't be live";
 +    }
 +
 +    @Test
 +    public void rangeTombstoneFilteringTest() throws Exception
 +    {
 +        CompactionManager.instance.disableAutoCompaction();
 +        Keyspace keyspace = Keyspace.open(KSNAME);
 +        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CFNAME);
 +
 +        // Inserting data
 +        String key = "k111";
 +        Mutation rm;
 +        ColumnFamily cf;
 +
 +        rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key));
 +        for (int i = 0; i < 40; i += 2)
 +            add(rm, i, 0);
 +        rm.apply();
 +
 +        rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key));
 +        cf = rm.addOrGet(CFNAME);
 +        delete(cf, 5, 10, 1);
 +        rm.apply();
 +
 +        rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key));
 +        cf = rm.addOrGet(CFNAME);
 +        delete(cf, 15, 20, 2);
 +        rm.apply();
 +
 +        cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(11), b(14),
false, Integer.MAX_VALUE, System.currentTimeMillis()));
 +        Collection<RangeTombstone> rt = rangeTombstones(cf);
 +        assertEquals(0, rt.size());
 +
 +        cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(11), b(15),
false, Integer.MAX_VALUE, System.currentTimeMillis()));
 +        rt = rangeTombstones(cf);
 +        assertEquals(1, rt.size());
 +
 +        cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(20), b(25),
false, Integer.MAX_VALUE, System.currentTimeMillis()));
 +        rt = rangeTombstones(cf);
 +        assertEquals(1, rt.size());
 +
 +        cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(12), b(25),
false, Integer.MAX_VALUE, System.currentTimeMillis()));
 +        rt = rangeTombstones(cf);
 +        assertEquals(1, rt.size());
 +
 +        cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(25), b(35),
false, Integer.MAX_VALUE, System.currentTimeMillis()));
 +        rt = rangeTombstones(cf);
 +        assertEquals(0, rt.size());
 +
 +        cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(1), b(40),
false, Integer.MAX_VALUE, System.currentTimeMillis()));
 +        rt = rangeTombstones(cf);
 +        assertEquals(2, rt.size());
 +
 +        cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(7), b(17),
false, Integer.MAX_VALUE, System.currentTimeMillis()));
 +        rt = rangeTombstones(cf);
 +        assertEquals(2, rt.size());
 +
 +        cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(5), b(20),
false, Integer.MAX_VALUE, System.currentTimeMillis()));
 +        rt = rangeTombstones(cf);
 +        assertEquals(2, rt.size());
 +
 +        cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(5), b(15),
false, Integer.MAX_VALUE, System.currentTimeMillis()));
 +        rt = rangeTombstones(cf);
 +        assertEquals(2, rt.size());
 +
 +        cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(1), b(2),
false, Integer.MAX_VALUE, System.currentTimeMillis()));
 +        rt = rangeTombstones(cf);
 +        assertEquals(0, rt.size());
 +
 +        cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(1), b(5),
false, Integer.MAX_VALUE, System.currentTimeMillis()));
 +        rt = rangeTombstones(cf);
 +        assertEquals(1, rt.size());
 +
 +        cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(1), b(10),
false, Integer.MAX_VALUE, System.currentTimeMillis()));
 +        rt = rangeTombstones(cf);
 +        assertEquals(1, rt.size());
 +
 +        cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(5), b(6),
false, Integer.MAX_VALUE, System.currentTimeMillis()));
 +        rt = rangeTombstones(cf);
 +        assertEquals(1, rt.size());
 +
 +        cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(17), b(20),
false, Integer.MAX_VALUE, System.currentTimeMillis()));
 +        rt = rangeTombstones(cf);
 +        assertEquals(1, rt.size());
 +
 +        cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(17), b(18),
false, Integer.MAX_VALUE, System.currentTimeMillis()));
 +        rt = rangeTombstones(cf);
 +        assertEquals(1, rt.size());
 +
 +        ColumnSlice[] slices = new ColumnSlice[]{new ColumnSlice( b(1), b(10)), new ColumnSlice(
b(16), b(20))};
 +        IDiskAtomFilter sqf = new SliceQueryFilter(slices, false, Integer.MAX_VALUE);
 +        cf = cfs.getColumnFamily( new QueryFilter(dk(key), CFNAME, sqf, System.currentTimeMillis())
);
 +        rt = rangeTombstones(cf);
 +        assertEquals(2, rt.size());
 +    }
 +
 +    private Collection<RangeTombstone> rangeTombstones(ColumnFamily cf)
 +    {
 +        List<RangeTombstone> tombstones = new ArrayList<RangeTombstone>();
 +        Iterators.addAll(tombstones, cf.deletionInfo().rangeIterator());
 +        return tombstones;
      }
  
+ 
+     @Test
+     public void test7810() throws ExecutionException, InterruptedException, IOException
+     {
 -        DatabaseDescriptor.setInMemoryCompactionLimit(0);
+         Keyspace ks = Keyspace.open(KSNAME);
+         ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME);
+         cfs.metadata.gcGraceSeconds(2);
+ 
+         String key = "7810";
 -        RowMutation rm;
 -        rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
++        Mutation rm;
++        rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key));
+         for (int i = 10; i < 20; i++)
+             add(rm, i, 0);
+         rm.apply();
+         cfs.forceBlockingFlush();
+ 
 -        rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
++        rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key));
+         ColumnFamily cf = rm.addOrGet(CFNAME);
+         cf.delete(new DeletionInfo(b(10),b(11), cfs.getComparator(), 1, 1));
+         rm.apply();
+         cfs.forceBlockingFlush();
+         Thread.sleep(5);
+         cfs.forceMajorCompaction();
+         assertEquals(8, Util.getColumnFamily(ks, Util.dk(key), CFNAME).getColumnCount());
+     }
+ 
      @Test
      public void test7808_1() throws ExecutionException, InterruptedException
      {


Mime
View raw message