cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yu...@apache.org
Subject [1/7] git commit: Merge branch 'cassandra-1.1' into trunk
Date Mon, 22 Oct 2012 18:55:18 GMT
Updated Branches:
  refs/heads/cassandra-1.1 f22e2c459 -> 95fb613bf
  refs/heads/trunk a58b87020 -> dd8a3c450


Merge branch 'cassandra-1.1' into trunk

Conflicts:
	src/java/org/apache/cassandra/service/StorageService.java
	src/java/org/apache/cassandra/thrift/CassandraServer.java


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/dd8a3c45
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/dd8a3c45
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/dd8a3c45

Branch: refs/heads/trunk
Commit: dd8a3c45072efbc0fea14e98552b4b5a9dab9de3
Parents: a58b870 95fb613
Author: Yuki Morishita <yukim@apache.org>
Authored: Mon Oct 22 13:53:11 2012 -0500
Committer: Yuki Morishita <yukim@apache.org>
Committed: Mon Oct 22 13:53:11 2012 -0500

----------------------------------------------------------------------
 CHANGES.txt                                        |    1 +
 .../apache/cassandra/thrift/CassandraServer.java   |   12 ++-
 .../cassandra/service/CassandraServerTest.java     |   91 ++++++++++-----
 3 files changed, 70 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/dd8a3c45/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 4026428,f309ef1..0f7caba
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -40,76 -5,8 +40,77 @@@ Merged from 1.1
     (CASSANDRA-4765)
   * fix wrong leveled compaction progress calculation (CASSANDRA-4807)
   * add a close() method to CRAR to prevent leaking file descriptors (CASSANDRA-4820)
+  * fix potential infinite loop in get_count (CASSANDRA-4833)
  
 +
 +1.2-beta1
 + * add atomic_batch_mutate (CASSANDRA-4542, -4635)
 + * increase default max_hint_window_in_ms to 3h (CASSANDRA-4632)
 + * include message initiation time to replicas so they can more
 +   accurately drop timed-out requests (CASSANDRA-2858)
 + * fix clientutil.jar dependencies (CASSANDRA-4566)
 + * optimize WriteResponse (CASSANDRA-4548)
 + * new metrics (CASSANDRA-4009)
 + * redesign KEYS indexes to avoid read-before-write (CASSANDRA-2897)
 + * debug tracing (CASSANDRA-1123)
 + * parallelize row cache loading (CASSANDRA-4282)
 + * Make compaction, flush JBOD-aware (CASSANDRA-4292)
 + * run local range scans on the read stage (CASSANDRA-3687)
 + * clean up ioexceptions (CASSANDRA-2116)
 + * add disk_failure_policy (CASSANDRA-2118)
 + * Introduce new json format with row level deletion (CASSANDRA-4054)
 + * remove redundant "name" column from schema_keyspaces (CASSANDRA-4433)
 + * improve "nodetool ring" handling of multi-dc clusters (CASSANDRA-3047)
 + * update NTS calculateNaturalEndpoints to be O(N log N) (CASSANDRA-3881)
 + * add UseCondCardMark XX jvm settings on jdk 1.7 (CASSANDRA-4366)
 + * split up rpc timeout by operation type (CASSANDRA-2819)
 + * rewrite key cache save/load to use only sequential i/o (CASSANDRA-3762)
 + * update MS protocol with a version handshake + broadcast address id
 +   (CASSANDRA-4311)
 + * multithreaded hint replay (CASSANDRA-4189)
 + * add inter-node message compression (CASSANDRA-3127)
 + * remove COPP (CASSANDRA-2479)
 + * Track tombstone expiration and compact when tombstone content is
 +   higher than a configurable threshold, default 20% (CASSANDRA-3442, 4234)
 + * update MurmurHash to version 3 (CASSANDRA-2975)
 + * (CLI) track elapsed time for `delete' operation (CASSANDRA-4060)
 + * (CLI) jline version is bumped to 1.0 to properly  support
 +   'delete' key function (CASSANDRA-4132)
 + * Save IndexSummary into new SSTable 'Summary' component (CASSANDRA-2392, 4289)
 + * Add support for range tombstones (CASSANDRA-3708)
 + * Improve MessagingService efficiency (CASSANDRA-3617)
 + * Avoid ID conflicts from concurrent schema changes (CASSANDRA-3794)
 + * Set thrift HSHA server thread limit to unlimited by default (CASSANDRA-4277)
 + * Avoids double serialization of CF id in RowMutation messages
 +   (CASSANDRA-4293)
 + * stream compressed sstables directly with java nio (CASSANDRA-4297)
 + * Support multiple ranges in SliceQueryFilter (CASSANDRA-3885)
 + * Add column metadata to system column families (CASSANDRA-4018)
 + * (cql3) Always use composite types by default (CASSANDRA-4329)
 + * (cql3) Add support for set, map and list (CASSANDRA-3647)
 + * Validate date type correctly (CASSANDRA-4441)
 + * (cql3) Allow definitions with only a PK (CASSANDRA-4361)
 + * (cql3) Add support for row key composites (CASSANDRA-4179)
 + * improve DynamicEndpointSnitch by using reservoir sampling (CASSANDRA-4038)
 + * (cql3) Add support for 2ndary indexes (CASSANDRA-3680)
 + * (cql3) fix defining more than one PK to be invalid (CASSANDRA-4477)
 + * remove schema agreement checking from all external APIs (Thrift, CQL and CQL3) (CASSANDRA-4487)
 + * add Murmur3Partitioner and make it default for new installations (CASSANDRA-3772, 4621)
 + * (cql3) update pseudo-map syntax to use map syntax (CASSANDRA-4497)
 + * Finer grained exceptions hierarchy and provides error code with exceptions (CASSANDRA-3979)
 + * Adds events push to binary protocol (CASSANDRA-4480)
 + * Rewrite nodetool help (CASSANDRA-2293)
 + * Make CQL3 the default for CQL (CASSANDRA-4640)
 + * update stress tool to be able to use CQL3 (CASSANDRA-4406)
 + * Accept all thrift update on CQL3 cf but don't expose their metadata (CASSANDRA-4377)
 + * Replace Throttle with Guava's RateLimiter for HintedHandOff (CASSANDRA-4541)
 + * fix counter add/get using CQL2 and CQL3 in stress tool (CASSANDRA-4633)
 + * Add sstable count per level to cfstats (CASSANDRA-4537)
 + * (cql3) Add ALTER KEYSPACE statement (CASSANDRA-4611)
 + * (cql3) Allow defining default consistency levels (CASSANDRA-4448)
 + * (cql3) Fix queries using LIMIT missing results (CASSANDRA-4579)
 + * fix cross-version gossip messaging (CASSANDRA-4576)
 +
  1.1.6
   * Wait for writes on synchronous read digest mismatch (CASSANDRA-4792)
   * fix commitlog replay for nanotime-infected sstables (CASSANDRA-4782)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/dd8a3c45/src/java/org/apache/cassandra/thrift/CassandraServer.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/thrift/CassandraServer.java
index 2c37ba6,39b57f3..d21d9a2
--- a/src/java/org/apache/cassandra/thrift/CassandraServer.java
+++ b/src/java/org/apache/cassandra/thrift/CassandraServer.java
@@@ -464,80 -414,48 +464,84 @@@ public class CassandraServer implement
          }
          else
          {
 -            pageSize = COUNT_PAGE_SIZE;
 +            logger.debug("get_count");
          }
  
 -        int totalCount = 0;
 -        List<ColumnOrSuperColumn> columns;
 -
 -        if (predicate.slice_range == null)
 +        try
          {
 -            predicate.slice_range = new SliceRange(ByteBufferUtil.EMPTY_BYTE_BUFFER,
 -                                                   ByteBufferUtil.EMPTY_BYTE_BUFFER,
 -                                                   false,
 -                                                   Integer.MAX_VALUE);
 -        }
 +            ClientState cState = state();
 +            cState.hasColumnFamilyAccess(column_parent.column_family, Permission.SELECT);
 +            Table table = Table.open(cState.getKeyspace());
 +            ColumnFamilyStore cfs = table.getColumnFamilyStore(column_parent.column_family);
  
 -        final int requestedCount = predicate.slice_range.count;
 -        int remaining = requestedCount;
 -        int pages = 0;
 -        while (true)
 -        {
 -            predicate.slice_range.count = Math.min(pageSize, Math.max(2, remaining)); //
fetch at least two columns
 -            columns = get_slice(key, column_parent, predicate, consistency_level);
 -            if (columns.isEmpty())
 -                break;
 +            if (predicate.column_names != null)
 +                return get_slice(key, column_parent, predicate, consistency_level).size();
  
 -            ByteBuffer firstName = getName(columns.get(0));
 -            int newColumns = pages == 0 || !firstName.equals(predicate.slice_range.start)
? columns.size() : columns.size() - 1;
 -            totalCount += newColumns;
 -            // if we over-counted, just return original limit
 -            if (totalCount > requestedCount)
 -                return requestedCount;
 -            remaining -= newColumns;
 -            pages++;
 -            // We're done if either:
 -            //   - We've querying the number of columns requested by the user
 -            //   - The last page wasn't full
 -            if (remaining == 0 || columns.size() < predicate.slice_range.count)
 -                break;
 +            int pageSize;
 +            // request by page if this is a large row
 +            if (cfs.getMeanColumns() > 0)
 +            {
 +                int averageColumnSize = (int) (cfs.getMeanRowSize() / cfs.getMeanColumns());
 +                pageSize = Math.min(COUNT_PAGE_SIZE,
 +                        DatabaseDescriptor.getInMemoryCompactionLimit() / averageColumnSize);
 +                pageSize = Math.max(2, pageSize);
 +                logger.debug("average row column size is {}; using pageSize of {}", averageColumnSize,
pageSize);
 +            }
              else
 -                predicate.slice_range.start = getName(columns.get(columns.size() - 1));
 -        }
 +            {
 +                pageSize = COUNT_PAGE_SIZE;
 +            }
 +
 +            int totalCount = 0;
 +            List<ColumnOrSuperColumn> columns;
  
 -        return totalCount;
 +            if (predicate.slice_range == null)
 +            {
 +                predicate.slice_range = new SliceRange(ByteBufferUtil.EMPTY_BYTE_BUFFER,
 +                        ByteBufferUtil.EMPTY_BYTE_BUFFER,
 +                        false,
 +                        Integer.MAX_VALUE);
 +            }
 +
-             int requestedCount = predicate.slice_range.count;
++            final int requestedCount = predicate.slice_range.count;
++            int remaining = requestedCount;
 +            int pages = 0;
 +            while (true)
 +            {
-                 predicate.slice_range.count = Math.min(pageSize, requestedCount);
++                predicate.slice_range.count = Math.min(pageSize, Math.max(2, remaining));
// fetch at least two columns
 +                columns = get_slice(key, column_parent, predicate, consistency_level);
 +                if (columns.isEmpty())
 +                    break;
 +
 +                ByteBuffer firstName = getName(columns.get(0));
 +                int newColumns = pages == 0 || !firstName.equals(predicate.slice_range.start)
? columns.size()
 +                        : columns.size() - 1;
 +
 +                totalCount += newColumns;
-                 requestedCount -= newColumns;
++                // if we over-counted, just return original limit
++                if (totalCount > requestedCount)
++                    return requestedCount;
++                remaining -= newColumns;
 +                pages++;
 +                // We're done if either:
 +                // - We've querying the number of columns requested by the user
 +                // - The last page wasn't full
-                 if (requestedCount == 0 || columns.size() < predicate.slice_range.count)
++                if (remaining == 0 || columns.size() < predicate.slice_range.count)
 +                    break;
 +                else
 +                    predicate.slice_range.start = getName(columns.get(columns.size() - 1));
 +            }
 +
 +            return totalCount;
 +        }
 +        catch (RequestValidationException e)
 +        {
 +            throw ThriftConversion.toThrift(e);
 +        }
 +        finally
 +        {
 +            Tracing.instance().stopSession();
 +        }
      }
  
      private static ByteBuffer getName(ColumnOrSuperColumn cosc)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/dd8a3c45/test/unit/org/apache/cassandra/service/CassandraServerTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/service/CassandraServerTest.java
index 495d697,48701de..af2d1c7
--- a/test/unit/org/apache/cassandra/service/CassandraServerTest.java
+++ b/test/unit/org/apache/cassandra/service/CassandraServerTest.java
@@@ -18,6 -18,6 +18,8 @@@
  */
  package org.apache.cassandra.service;
  
++import java.net.InetSocketAddress;
++
  import org.junit.Test;
  
  import org.apache.cassandra.SchemaLoader;
@@@ -24,37 -31,58 +33,59 @@@ import org.apache.cassandra.utils.ByteB
  
  public class CassandraServerTest extends SchemaLoader
  {
+     /**
+      * test get_count() to work correctly with 'count' settings around page size.
+      * (CASSANDRA-4833)
+      */
      @Test
-     public void test_get_column() throws Throwable {
-         /*
-         CassandraServer server = new CassandraServer();
-         server.start();
- 
-         try {
-             Column c1 = column("c1", "0", 0L);
-             Column c2 = column("c2", "0", 0L);
-             List<Column> columns = new ArrayList<Column>();
-             columns.add(c1);
-             columns.add(c2);
-             Map<String, List<Column>> cfmap = new HashMap<String, List<Column>>();
-             cfmap.put("Standard1", columns);
-             cfmap.put("Standard2", columns);
- 
-             BatchMutation m = new BatchMutation("Keyspace1", "key1", cfmap);
-             server.batch_insert(m, 1);
- 
-             Column column;
-             column = server.get_column("Keyspace1", "key1", "Standard1:c2");
-             assert column.value.equals("0");
- 
-             column = server.get_column("Keyspace1", "key1", "Standard2:c2");
-             assert column.value.equals("0");
- 
-             ArrayList<Column> Columns = server.get_slice_strong("Keyspace1", "key1",
"Standard1", -1, -1);
-             assert Columns.size() == 2;
-         } finally {
-             server.shutdown();
+     public void test_get_count() throws Exception
+     {
+         Schema.instance.clear(); // Schema are now written on disk and will be reloaded
+         new EmbeddedCassandraService().start();
++        ThriftSessionManager.instance.setCurrentSocket(new InetSocketAddress(9160));
+ 
+         DecoratedKey key = Util.dk("testkey");
+         for (int i = 0; i < 3050; i++)
+         {
+             RowMutation rm = new RowMutation("Keyspace1", key.key);
+             rm.add(new QueryPath("Standard1", null, ByteBufferUtil.bytes(String.valueOf(i))),
+                           ByteBufferUtil.EMPTY_BYTE_BUFFER,
+                           System.currentTimeMillis());
+             rm.apply();
          }
-         */
+ 
+         CassandraServer server = new CassandraServer();
+         server.set_keyspace("Keyspace1");
+ 
+         // same as page size
+         int count = server.get_count(key.key, new ColumnParent("Standard1"), predicateWithCount(1024),
ConsistencyLevel.ONE);
+         assert count == 1024 : "expected 1024 but was " + count;
+ 
+         // 1 above page size
+         count = server.get_count(key.key, new ColumnParent("Standard1"), predicateWithCount(1025),
ConsistencyLevel.ONE);
+         assert count == 1025 : "expected 1025 but was " + count;
+ 
+         // above number of columns
+         count = server.get_count(key.key, new ColumnParent("Standard1"), predicateWithCount(4000),
ConsistencyLevel.ONE);
+         assert count == 3050 : "expected 3050 but was " + count;
+ 
+         // same as number of columns
+         count = server.get_count(key.key, new ColumnParent("Standard1"), predicateWithCount(3050),
ConsistencyLevel.ONE);
+         assert count == 3050 : "expected 3050 but was " + count;
+ 
+         // 1 above number of columns
+         count = server.get_count(key.key, new ColumnParent("Standard1"), predicateWithCount(3051),
ConsistencyLevel.ONE);
+         assert count == 3050 : "expected 3050 but was " + count;
+     }
+ 
+     private SlicePredicate predicateWithCount(int count)
+     {
+         SliceRange range = new SliceRange();
+         range.setStart("".getBytes());
+         range.setFinish("".getBytes());
+         range.setCount(count);
+         SlicePredicate predicate = new SlicePredicate();
+         predicate.setSlice_range(range);
+         return predicate;
      }
  }


Mime
View raw message