cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yu...@apache.org
Subject [5/6] cassandra git commit: Merge branch 'cassandra-2.0' into cassandra-2.1
Date Thu, 05 Mar 2015 17:52:50 GMT
Merge branch 'cassandra-2.0' into cassandra-2.1


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ecf48dd7
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ecf48dd7
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ecf48dd7

Branch: refs/heads/cassandra-2.1
Commit: ecf48dd79d29ba80008afa26fd40c3fc6d9509ac
Parents: b1c2536 fb67c41
Author: Yuki Morishita <yukim@apache.org>
Authored: Thu Mar 5 11:48:22 2015 -0600
Committer: Yuki Morishita <yukim@apache.org>
Committed: Thu Mar 5 11:48:22 2015 -0600

----------------------------------------------------------------------
 CHANGES.txt                                     |   2 +
 .../cql3/statements/UpdateStatement.java        |  29 ++++-
 .../cassandra/io/sstable/CQLSSTableWriter.java  |  56 +++++----
 .../io/sstable/CQLSSTableWriterClientTest.java  | 116 +++++++++++++++++++
 .../io/sstable/CQLSSTableWriterTest.java        |  21 +++-
 5 files changed, 195 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/ecf48dd7/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index aa2e1f9,faa14d5..ea79e22
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -42,105 -10,10 +42,107 @@@ Merged from 2.0
     table with ASC ordering and paging (CASSANDRA-8767)
   * AssertionError: "Memory was freed" when running cleanup (CASSANDRA-8716)
   * Make it possible to set max_sstable_age to fractional days (CASSANDRA-8406)
 - * Fix memory leak in SSTableSimple*Writer and SSTableReader.validate()
 -   (CASSANDRA-8748)
   * Fix some multi-column relations with indexes on some clustering
     columns (CASSANDRA-8275)
 + * Fix memory leak in SSTableSimple*Writer and SSTableReader.validate()
 +   (CASSANDRA-8748)
 + * Throw OOM if allocating memory fails to return a valid pointer (CASSANDRA-8726)
 + * Fix SSTableSimpleUnsortedWriter ConcurrentModificationException (CASSANDRA-8619)
 + * 'nodetool info' prints exception against older node (CASSANDRA-8796)
 + * Ensure SSTableSimpleUnsortedWriter.close() terminates if
 +   disk writer has crashed (CASSANDRA-8807)
++ * Fix CQLSSTableWriter throwing exception and spawning threads
++   (CASSANDRA-8808)
 +
 +
 +2.1.3
 + * Fix HSHA/offheap_objects corruption (CASSANDRA-8719)
 + * Upgrade libthrift to 0.9.2 (CASSANDRA-8685)
 + * Don't use the shared ref in sstableloader (CASSANDRA-8704)
 + * Purge internal prepared statements if related tables or
 +   keyspaces are dropped (CASSANDRA-8693)
 + * (cqlsh) Handle unicode BOM at start of files (CASSANDRA-8638)
 + * Stop compactions before exiting offline tools (CASSANDRA-8623)
 + * Update tools/stress/README.txt to match current behaviour (CASSANDRA-7933)
 + * Fix schema from Thrift conversion with empty metadata (CASSANDRA-8695)
 + * Safer Resource Management (CASSANDRA-7705)
 + * Make sure we compact highly overlapping cold sstables with
 +   STCS (CASSANDRA-8635)
 + * rpc_interface and listen_interface generate NPE on startup when specified
 +   interface doesn't exist (CASSANDRA-8677)
 + * Fix ArrayIndexOutOfBoundsException in nodetool cfhistograms (CASSANDRA-8514)
 + * Switch from yammer metrics for nodetool cf/proxy histograms (CASSANDRA-8662)
 + * Make sure we don't add tmplink files to the compaction
 +   strategy (CASSANDRA-8580)
 + * (cqlsh) Handle maps with blob keys (CASSANDRA-8372)
 + * (cqlsh) Handle DynamicCompositeType schemas correctly (CASSANDRA-8563)
 + * Duplicate rows returned when in clause has repeated values (CASSANDRA-6706)
 + * Add tooling to detect hot partitions (CASSANDRA-7974)
 + * Fix cassandra-stress user-mode truncation of partition generation (CASSANDRA-8608)
 + * Only stream from unrepaired sstables during inc repair (CASSANDRA-8267)
 + * Don't allow starting multiple inc repairs on the same sstables (CASSANDRA-8316)
 + * Invalidate prepared BATCH statements when related tables
 +   or keyspaces are dropped (CASSANDRA-8652)
 + * Fix missing results in secondary index queries on collections
 +   with ALLOW FILTERING (CASSANDRA-8421)
 + * Expose EstimatedHistogram metrics for range slices (CASSANDRA-8627)
 + * (cqlsh) Escape clqshrc passwords properly (CASSANDRA-8618)
 + * Fix NPE when passing wrong argument in ALTER TABLE statement (CASSANDRA-8355)
 + * Pig: Refactor and deprecate CqlStorage (CASSANDRA-8599)
 + * Don't reuse the same cleanup strategy for all sstables (CASSANDRA-8537)
 + * Fix case-sensitivity of index name on CREATE and DROP INDEX
 +   statements (CASSANDRA-8365)
 + * Better detection/logging for corruption in compressed sstables (CASSANDRA-8192)
 + * Use the correct repairedAt value when closing writer (CASSANDRA-8570)
 + * (cqlsh) Handle a schema mismatch being detected on startup (CASSANDRA-8512)
 + * Properly calculate expected write size during compaction (CASSANDRA-8532)
 + * Invalidate affected prepared statements when a table's columns
 +   are altered (CASSANDRA-7910)
 + * Stress - user defined writes should populate sequentally (CASSANDRA-8524)
 + * Fix regression in SSTableRewriter causing some rows to become unreadable 
 +   during compaction (CASSANDRA-8429)
 + * Run major compactions for repaired/unrepaired in parallel (CASSANDRA-8510)
 + * (cqlsh) Fix compression options in DESCRIBE TABLE output when compression
 +   is disabled (CASSANDRA-8288)
 + * (cqlsh) Fix DESCRIBE output after keyspaces are altered (CASSANDRA-7623)
 + * Make sure we set lastCompactedKey correctly (CASSANDRA-8463)
 + * (cqlsh) Fix output of CONSISTENCY command (CASSANDRA-8507)
 + * (cqlsh) Fixed the handling of LIST statements (CASSANDRA-8370)
 + * Make sstablescrub check leveled manifest again (CASSANDRA-8432)
 + * Check first/last keys in sstable when giving out positions (CASSANDRA-8458)
 + * Disable mmap on Windows (CASSANDRA-6993)
 + * Add missing ConsistencyLevels to cassandra-stress (CASSANDRA-8253)
 + * Add auth support to cassandra-stress (CASSANDRA-7985)
 + * Fix ArrayIndexOutOfBoundsException when generating error message
 +   for some CQL syntax errors (CASSANDRA-8455)
 + * Scale memtable slab allocation logarithmically (CASSANDRA-7882)
 + * cassandra-stress simultaneous inserts over same seed (CASSANDRA-7964)
 + * Reduce cassandra-stress sampling memory requirements (CASSANDRA-7926)
 + * Ensure memtable flush cannot expire commit log entries from its future (CASSANDRA-8383)
 + * Make read "defrag" async to reclaim memtables (CASSANDRA-8459)
 + * Remove tmplink files for offline compactions (CASSANDRA-8321)
 + * Reduce maxHintsInProgress (CASSANDRA-8415)
 + * BTree updates may call provided update function twice (CASSANDRA-8018)
 + * Release sstable references after anticompaction (CASSANDRA-8386)
 + * Handle abort() in SSTableRewriter properly (CASSANDRA-8320)
 + * Fix high size calculations for prepared statements (CASSANDRA-8231)
 + * Centralize shared executors (CASSANDRA-8055)
 + * Fix filtering for CONTAINS (KEY) relations on frozen collection
 +   clustering columns when the query is restricted to a single
 +   partition (CASSANDRA-8203)
 + * Do more aggressive entire-sstable TTL expiry checks (CASSANDRA-8243)
 + * Add more log info if readMeter is null (CASSANDRA-8238)
 + * add check of the system wall clock time at startup (CASSANDRA-8305)
 + * Support for frozen collections (CASSANDRA-7859)
 + * Fix overflow on histogram computation (CASSANDRA-8028)
 + * Have paxos reuse the timestamp generation of normal queries (CASSANDRA-7801)
 + * Fix incremental repair not remove parent session on remote (CASSANDRA-8291)
 + * Improve JBOD disk utilization (CASSANDRA-7386)
 + * Log failed host when preparing incremental repair (CASSANDRA-8228)
 + * Force config client mode in CQLSSTableWriter (CASSANDRA-8281)
 + * Fix sstableupgrade throws exception (CASSANDRA-8688)
 + * Fix hang when repairing empty keyspace (CASSANDRA-8694)
 +Merged from 2.0:
   * Fix IllegalArgumentException in dynamic snitch (CASSANDRA-8448)
   * Add support for UPDATE ... IF EXISTS (CASSANDRA-8610)
   * Fix reversal of list prepends (CASSANDRA-8733)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ecf48dd7/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
index c35f985,f34edaf..63f87c0
--- a/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java
@@@ -48,9 -47,25 +48,20 @@@ public class UpdateStatement extends Mo
          return true;
      }
  
-     public void addUpdateForKey(ColumnFamily cf, ByteBuffer key, Composite prefix, UpdateParameters
params)
 -    public void addUpdateForKey(ColumnFamily cf, ByteBuffer key, ColumnNameBuilder builder,
UpdateParameters params)
--    throws InvalidRequestException
++    public void addUpdateForKey(ColumnFamily cf,
++                                ByteBuffer key,
++                                Composite prefix,
++                                UpdateParameters params) throws InvalidRequestException
+     {
 -        addUpdateForKey(cf, key, builder, params, true);
++        addUpdateForKey(cf, key, prefix, params, true);
+     }
+ 
+     public void addUpdateForKey(ColumnFamily cf,
+                                 ByteBuffer key,
 -                                ColumnNameBuilder builder,
++                                Composite prefix,
+                                 UpdateParameters params,
+                                 boolean validateIndexedColumns) throws InvalidRequestException
      {
 -        CFDefinition cfDef = cfm.getCfDef();
 -
 -        if (builder.getLength() > FBUtilities.MAX_UNSIGNED_SHORT)
 -            throw new InvalidRequestException(String.format("The sum of all clustering columns
is too long (%s > %s)",
 -                                                            builder.getLength(),
 -                                                            FBUtilities.MAX_UNSIGNED_SHORT));
 -
          // Inserting the CQL row marker (see #4361)
          // We always need to insert a marker for INSERT, because of the following situation:
          //   CREATE TABLE t ( k int PRIMARY KEY, c text );
@@@ -95,9 -112,23 +106,23 @@@
          else
          {
              for (Operation update : updates)
 -                update.execute(key, cf, builder.copy(), params);
 +                update.execute(key, cf, prefix, params);
          }
  
+         // validateIndexedColumns trigger a call to Keyspace.open() which we want to be
able to avoid in some case
+         //(e.g. when using CQLSSTableWriter)
+         if (validateIndexedColumns)
+             validateIndexedColumns(cf);
+     }
+ 
+     /**
+      * Checks that the value of the indexed columns is valid.
+      *
+      * @param cf the column family
+      * @throws InvalidRequestException if one of the values is invalid
+      */
+     private void validateIndexedColumns(ColumnFamily cf) throws InvalidRequestException
+     {
          SecondaryIndexManager indexManager = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfId).indexManager;
          if (indexManager.hasIndexes())
          {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ecf48dd7/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java
index 3e27ff9,fb4c186..b4bef7d
--- a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java
@@@ -34,8 -34,6 +34,7 @@@ import org.apache.cassandra.cql3.statem
  import org.apache.cassandra.cql3.*;
  import org.apache.cassandra.config.*;
  import org.apache.cassandra.db.*;
- import org.apache.cassandra.db.commitlog.CommitLog;
 +import org.apache.cassandra.db.composites.Composite;
  import org.apache.cassandra.db.marshal.AbstractType;
  import org.apache.cassandra.dht.IPartitioner;
  import org.apache.cassandra.dht.Murmur3Partitioner;
@@@ -218,11 -216,11 +217,11 @@@ public class CQLSSTableWriter implement
  
          try
          {
 -            for (ByteBuffer key: keys)
 +            for (ByteBuffer key : keys)
              {
 -                if (writer.currentKey() == null || !key.equals(writer.currentKey().key))
 +                if (writer.currentKey() == null || !key.equals(writer.currentKey().getKey()))
                      writer.newRow(key);
-                 insert.addUpdateForKey(writer.currentColumnFamily(), key, clusteringPrefix,
params);
+                 insert.addUpdateForKey(writer.currentColumnFamily(), key, clusteringPrefix,
params, false);
              }
              return this;
          }
@@@ -360,19 -350,11 +351,11 @@@
                      KSMetaData ksm = Schema.instance.getKSMetaData(this.schema.ksName);
                      if (ksm == null)
                      {
-                         ksm = KSMetaData.newKeyspace(this.schema.ksName,
-                                 AbstractReplicationStrategy.getClass("org.apache.cassandra.locator.SimpleStrategy"),
-                                 ImmutableMap.of("replication_factor", "1"),
-                                 true,
-                                 Collections.singleton(this.schema));
-                         Schema.instance.load(ksm);
 -                        createKeyspaceWithColumnFamily(this.schema);
++                        createKeyspaceWithTable(this.schema);
                      }
                      else if (Schema.instance.getCFMetaData(this.schema.ksName, this.schema.cfName)
== null)
                      {
-                         Schema.instance.load(this.schema);
-                         ksm = KSMetaData.cloneWith(ksm, Iterables.concat(ksm.cfMetaData().values(),
Collections.singleton(this.schema)));
-                         Schema.instance.setKeyspaceDefinition(ksm);
-                         Keyspace.open(ksm.name).initCf(this.schema.cfId, this.schema.cfName,
false);
 -                        addColumnFamilyToKeyspace(ksm, this.schema);
++                        addTableToKeyspace(ksm, this.schema);
                      }
                      return this;
                  }
@@@ -384,6 -366,35 +367,35 @@@
          }
  
          /**
+          * Adds the specified column family to the specified keyspace.
+          *
+          * @param ksm the keyspace meta data
+          * @param cfm the column family meta data
+          */
 -        private static void addColumnFamilyToKeyspace(KSMetaData ksm, CFMetaData cfm)
++        private static void addTableToKeyspace(KSMetaData ksm, CFMetaData cfm)
+         {
+             ksm = KSMetaData.cloneWith(ksm, Iterables.concat(ksm.cfMetaData().values(),
Collections.singleton(cfm)));
+             Schema.instance.load(cfm);
+             Schema.instance.setKeyspaceDefinition(ksm);
+         }
+ 
+         /**
+          * Creates a keyspace for the specified column family.
+          *
+          * @param cfm the column family
+          * @throws ConfigurationException if a problem occurs while creating the keyspace.
+          */
 -        private static void createKeyspaceWithColumnFamily(CFMetaData cfm) throws ConfigurationException
++        private static void createKeyspaceWithTable(CFMetaData cfm) throws ConfigurationException
+         {
+             KSMetaData ksm = KSMetaData.newKeyspace(cfm.ksName,
+                                                     AbstractReplicationStrategy.getClass("org.apache.cassandra.locator.SimpleStrategy"),
+                                                     ImmutableMap.of("replication_factor",
"1"),
+                                                     true,
+                                                     Collections.singleton(cfm));
+             Schema.instance.load(ksm);
+         }
+ 
+         /**
           * The partitioner to use.
           * <p>
           * By default, {@code Murmur3Partitioner} will be used. If this is not the partitioner
used

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ecf48dd7/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
index 61b8b83,0922502..a2f1bcc
--- a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
@@@ -28,7 -30,8 +30,9 @@@ import org.junit.AfterClass
  import org.junit.BeforeClass;
  import org.junit.Test;
  
++import org.apache.cassandra.SchemaLoader;
  import org.apache.cassandra.config.CFMetaData;
+ import org.apache.cassandra.config.Config;
  import org.apache.cassandra.config.Schema;
  import org.apache.cassandra.cql3.QueryProcessor;
  import org.apache.cassandra.cql3.UntypedResultSet;
@@@ -47,7 -49,6 +51,8 @@@ public class CQLSSTableWriterTes
      @BeforeClass
      public static void setup() throws Exception
      {
++        SchemaLoader.cleanupAndLeaveDirs();
 +        Keyspace.setInitialized();
          StorageService.instance.initServer();
      }
  
@@@ -248,7 -255,7 +259,7 @@@
  
          loader.stream().get();
  
-         UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * FROM cql_keyspace.table2;");
 -        UntypedResultSet rs = QueryProcessor.processInternal("SELECT * FROM cql_keyspace2.table2;");
++        UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * FROM cql_keyspace2.table2;");
          assertEquals(threads.length * NUMBER_WRITES_IN_RUNNABLE, rs.size());
      }
  }


Mime
View raw message