Return-Path: X-Original-To: apmail-cassandra-commits-archive@www.apache.org Delivered-To: apmail-cassandra-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 58722FCC5 for ; Thu, 4 Apr 2013 16:31:51 +0000 (UTC) Received: (qmail 63082 invoked by uid 500); 4 Apr 2013 16:31:50 -0000 Delivered-To: apmail-cassandra-commits-archive@cassandra.apache.org Received: (qmail 62756 invoked by uid 500); 4 Apr 2013 16:31:49 -0000 Mailing-List: contact commits-help@cassandra.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cassandra.apache.org Delivered-To: mailing list commits@cassandra.apache.org Received: (qmail 62671 invoked by uid 99); 4 Apr 2013 16:31:47 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 04 Apr 2013 16:31:47 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 067318379F5; Thu, 4 Apr 2013 16:31:47 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: slebresne@apache.org To: commits@cassandra.apache.org Date: Thu, 04 Apr 2013 16:31:48 -0000 Message-Id: <3065e72237e847aa8cd6b6bc17c017a1@git.apache.org> In-Reply-To: <19e77175a6a247a1ba42bd9222c2001d@git.apache.org> References: <19e77175a6a247a1ba42bd9222c2001d@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [3/3] git commit: Support indexes on composite column components Support indexes on composite column components patch by slebresne; reviewed by iamaleksey for CASSANDRA-5125 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/a950b925 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/a950b925 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/a950b925 Branch: refs/heads/trunk Commit: a950b9257f4c92d067eb5e1d437096699123ac9b Parents: 2739248 Author: Sylvain Lebresne Authored: Tue Jan 15 16:21:53 2013 +0100 Committer: Sylvain Lebresne Committed: Thu Apr 4 18:10:35 2013 +0200 ---------------------------------------------------------------------- build.xml | 2 +- .../apache/cassandra/cache/AutoSavingCache.java | 4 +- src/java/org/apache/cassandra/config/Avro.java | 44 ++- .../org/apache/cassandra/config/CFMetaData.java | 451 +++++++++++---- .../apache/cassandra/config/ColumnDefinition.java | 140 +++-- .../org/apache/cassandra/config/KSMetaData.java | 8 +- .../apache/cassandra/cql/AlterTableStatement.java | 19 +- .../cassandra/cql/CreateColumnFamilyStatement.java | 4 +- .../apache/cassandra/cql/DropIndexStatement.java | 4 +- .../org/apache/cassandra/cql/QueryProcessor.java | 8 +- .../org/apache/cassandra/cql3/CFDefinition.java | 144 ++---- .../apache/cassandra/cql3/ColumnNameBuilder.java | 15 + .../cql3/statements/AlterTableStatement.java | 54 +-- .../statements/CreateColumnFamilyStatement.java | 12 +- .../cql3/statements/CreateIndexStatement.java | 21 +- .../cql3/statements/DropIndexStatement.java | 6 +- .../cassandra/cql3/statements/SelectStatement.java | 225 +++++--- .../org/apache/cassandra/db/ColumnFamilyStore.java | 6 +- src/java/org/apache/cassandra/db/EmptyColumns.java | 3 +- .../apache/cassandra/db/filter/ExtendedFilter.java | 57 ++- .../cassandra/db/filter/IDiskAtomFilter.java | 2 + .../cassandra/db/filter/NamesQueryFilter.java | 10 + .../cassandra/db/filter/SliceQueryFilter.java | 4 +- .../AbstractSimplePerColumnSecondaryIndex.java | 22 +- .../apache/cassandra/db/index/SecondaryIndex.java | 28 +- .../cassandra/db/index/SecondaryIndexManager.java | 118 ++-- .../cassandra/db/index/SecondaryIndexSearcher.java | 31 +- .../db/index/composites/CompositesIndex.java | 140 ++++-- .../composites/CompositesIndexOnClusteringKey.java | 116 ++++ .../composites/CompositesIndexOnPartitionKey.java | 105 ++++ .../index/composites/CompositesIndexOnRegular.java | 105 ++++ .../db/index/composites/CompositesSearcher.java | 184 ++---- .../apache/cassandra/db/index/keys/KeysIndex.java | 15 +- .../cassandra/db/index/keys/KeysSearcher.java | 32 +- .../apache/cassandra/db/marshal/AbstractType.java | 20 + .../apache/cassandra/db/marshal/CompositeType.java | 25 + .../apache/cassandra/service/MigrationManager.java | 4 +- .../apache/cassandra/thrift/CassandraServer.java | 2 +- .../org/apache/cassandra/utils/ByteBufferUtil.java | 10 + test/unit/org/apache/cassandra/SchemaLoader.java | 22 +- test/unit/org/apache/cassandra/cli/CliTest.java | 1 - .../apache/cassandra/config/CFMetaDataTest.java | 9 +- .../cassandra/config/ColumnDefinitionTest.java | 15 +- .../unit/org/apache/cassandra/config/DefsTest.java | 44 +- .../apache/cassandra/db/ColumnFamilyStoreTest.java | 4 +- .../cassandra/thrift/ThriftValidationTest.java | 7 +- 46 files changed, 1489 insertions(+), 813 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/a950b925/build.xml ---------------------------------------------------------------------- diff --git a/build.xml b/build.xml index 74f5a7b..86178c3 100644 --- a/build.xml +++ b/build.xml @@ -25,7 +25,7 @@ - + http://git-wip-us.apache.org/repos/asf/cassandra/blob/a950b925/src/java/org/apache/cassandra/cache/AutoSavingCache.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cache/AutoSavingCache.java b/src/java/org/apache/cassandra/cache/AutoSavingCache.java index 072385e..6a7e17d 100644 --- a/src/java/org/apache/cassandra/cache/AutoSavingCache.java +++ b/src/java/org/apache/cassandra/cache/AutoSavingCache.java @@ -30,10 +30,12 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyType; import org.apache.cassandra.db.Table; import org.apache.cassandra.db.compaction.CompactionInfo; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.OperationType; +import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.util.FileUtils; @@ -197,7 +199,7 @@ public class AutoSavingCache extends InstrumentingCachesingletonList(cf.key_alias)); } + + if (cf.key_alias != null) + newCFMD.addOrReplaceColumnDefinition(ColumnDefinition.partitionKeyDef(cf.key_alias, keyValidator, null)); if (cf.column_aliases != null) - newCFMD.columnAliases(new ArrayList(cf.column_aliases)); // fix avro stupidity - if (cf.value_alias != null) { newCFMD.valueAlias(cf.value_alias); } + { + if (comparator instanceof CompositeType) + { + List> components = ((CompositeType)comparator).types; + for (int i = 0; i < cf.column_aliases.size(); ++i) + if (cf.column_aliases.get(i) != null) + newCFMD.addOrReplaceColumnDefinition(ColumnDefinition.clusteringKeyDef(cf.column_aliases.get(i), components.get(i), i)); + } + else + { + assert cf.column_aliases.size() <= 1; + if (cf.column_aliases.get(0) != null) + newCFMD.addOrReplaceColumnDefinition(ColumnDefinition.clusteringKeyDef(cf.column_aliases.get(0), comparator, null)); + } + } + if (cf.value_alias != null) + newCFMD.addOrReplaceColumnDefinition(ColumnDefinition.compactValueDef(cf.value_alias, validator)); + if (cf.compaction_strategy != null) { try @@ -211,11 +230,28 @@ public class Avro try { AbstractType validatorType = TypeParser.parse(cd.validation_class); - return new ColumnDefinition(ByteBufferUtil.clone(cd.name), validatorType, index_type, ColumnDefinition.getStringMap(cd.index_options), index_name, null); + ColumnDefinition def = ColumnDefinition.regularDef(ByteBufferUtil.clone(cd.name), validatorType, null); + def.setIndexName(index_name); + def.setIndexType(index_type, getStringMap(cd.index_options)); + return def; } catch (RequestValidationException e) { throw new RuntimeException(e); } } + + public static Map getStringMap(Map charMap) + { + if (charMap == null) + return null; + + Map stringMap = new HashMap(); + + for (Map.Entry entry : charMap.entrySet()) + stringMap.put(entry.getKey().toString(), entry.getValue().toString()); + + + return stringMap; + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/a950b925/src/java/org/apache/cassandra/config/CFMetaData.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/CFMetaData.java b/src/java/org/apache/cassandra/config/CFMetaData.java index 74228a1..417929a 100644 --- a/src/java/org/apache/cassandra/config/CFMetaData.java +++ b/src/java/org/apache/cassandra/config/CFMetaData.java @@ -26,6 +26,7 @@ import java.util.*; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Objects; +import com.google.common.collect.AbstractIterator; import com.google.common.collect.MapDifference; import com.google.common.collect.Maps; import org.apache.commons.lang.ArrayUtils; @@ -158,6 +159,7 @@ public final class CFMetaData + "index_options text," + "index_name text," + "component_index int," + + "type text," + "PRIMARY KEY(keyspace_name, columnfamily_name, column_name)" + ") WITH COMMENT='ColumnFamily column attributes' AND gc_grace_seconds=8640"); @@ -346,10 +348,6 @@ public final class CFMetaData private volatile AbstractType keyValidator = BytesType.instance; private volatile int minCompactionThreshold = DEFAULT_MIN_COMPACTION_THRESHOLD; private volatile int maxCompactionThreshold = DEFAULT_MAX_COMPACTION_THRESHOLD; - // Both those aliases list can be null padded if only some of the position have been given an alias through ALTER TABLE .. RENAME - private volatile List keyAliases = new ArrayList(); - private volatile List columnAliases = new ArrayList(); - private volatile ByteBuffer valueAlias = null; private volatile Double bloomFilterFpChance = null; private volatile Caching caching = DEFAULT_CACHING_STRATEGY; private volatile int indexInterval = DEFAULT_INDEX_INTERVAL; @@ -358,7 +356,19 @@ public final class CFMetaData private volatile SpeculativeRetry speculativeRetry = DEFAULT_SPECULATIVE_RETRY; private volatile boolean populateIoCacheOnFlush = DEFAULT_POPULATE_IO_CACHE_ON_FLUSH; - volatile Map column_metadata = new HashMap(); + /* + * All CQL3 columns definition are stored in the column_metadata map. + * On top of that, we keep separated collection of each kind of definition, to + * 1) allow easy access to each kind and 2) for the partition key and + * clustering key ones, those list are ordered by the "component index" of the + * elements. + */ + private volatile Map column_metadata = new HashMap(); + private volatile List partitionKeyColumns; // Always of size keyValidator.componentsCount, null padded if necessary + private volatile List clusteringKeyColumns; // Of size comparator.componentsCount or comparator.componentsCount -1, null padded if necessary + private volatile Set regularColumns; + private volatile ColumnDefinition compactValueColumn; + public volatile Class compactionStrategyClass = DEFAULT_COMPACTION_STRATEGY_CLASS; public volatile Map compactionStrategyOptions = new HashMap(); @@ -378,9 +388,6 @@ public final class CFMetaData public CFMetaData keyValidator(AbstractType prop) {keyValidator = prop; updateCfDef(); return this;} public CFMetaData minCompactionThreshold(int prop) {minCompactionThreshold = prop; return this;} public CFMetaData maxCompactionThreshold(int prop) {maxCompactionThreshold = prop; return this;} - public CFMetaData keyAliases(List prop) {keyAliases = prop; updateCfDef(); return this;} - public CFMetaData columnAliases(List prop) {columnAliases = prop; updateCfDef(); return this;} - public CFMetaData valueAlias(ByteBuffer prop) {valueAlias = prop; updateCfDef(); return this;} public CFMetaData columnMetadata(Map prop) {column_metadata = prop; updateCfDef(); return this;} public CFMetaData compactionStrategyClass(Class prop) {compactionStrategyClass = prop; return this;} public CFMetaData compactionStrategyOptions(Map prop) {compactionStrategyOptions = prop; return this;} @@ -530,9 +537,6 @@ public final class CFMetaData .keyValidator(oldCFMD.keyValidator) .minCompactionThreshold(oldCFMD.minCompactionThreshold) .maxCompactionThreshold(oldCFMD.maxCompactionThreshold) - .keyAliases(new ArrayList(oldCFMD.keyAliases)) - .columnAliases(new ArrayList(oldCFMD.columnAliases)) - .valueAlias(oldCFMD.valueAlias) .columnMetadata(clonedColumns) .compactionStrategyClass(oldCFMD.compactionStrategyClass) .compactionStrategyOptions(oldCFMD.compactionStrategyOptions) @@ -630,35 +634,40 @@ public final class CFMetaData // Used by CQL2 only. public ByteBuffer getKeyName() { - if (keyAliases.size() > 1) + if (partitionKeyColumns.size() > 1) throw new IllegalStateException("Cannot acces column family with composite key from CQL < 3.0.0"); - return keyAliases.isEmpty() ? DEFAULT_KEY_NAME : keyAliases.get(0); + return partitionKeyColumns.get(0) == null ? DEFAULT_KEY_NAME : partitionKeyColumns.get(0).name; } - public List getKeyAliases() + public CompressionParameters compressionParameters() { - return keyAliases; + return compressionParameters; } - public List getColumnAliases() + public Collection allColumns() { - return columnAliases; + return column_metadata.values(); } - public ByteBuffer getValueAlias() + public List partitionKeyColumns() { - return valueAlias; + return partitionKeyColumns; } - public CompressionParameters compressionParameters() + public List clusteringKeyColumns() { - return compressionParameters; + return clusteringKeyColumns; + } + + public Set regularColumns() + { + return regularColumns; } - public Map getColumn_metadata() + public ColumnDefinition compactValueColumn() { - return Collections.unmodifiableMap(column_metadata); + return compactValueColumn; } public double getBloomFilterFpChance() @@ -722,9 +731,6 @@ public final class CFMetaData .append(maxCompactionThreshold, rhs.maxCompactionThreshold) .append(cfId, rhs.cfId) .append(column_metadata, rhs.column_metadata) - .append(keyAliases, rhs.keyAliases) - .append(columnAliases, rhs.columnAliases) - .append(valueAlias, rhs.valueAlias) .append(compactionStrategyClass, rhs.compactionStrategyClass) .append(compactionStrategyOptions, rhs.compactionStrategyOptions) .append(compressionParameters, rhs.compressionParameters) @@ -756,9 +762,6 @@ public final class CFMetaData .append(maxCompactionThreshold) .append(cfId) .append(column_metadata) - .append(keyAliases) - .append(columnAliases) - .append(valueAlias) .append(compactionStrategyClass) .append(compactionStrategyOptions) .append(compressionParameters) @@ -836,8 +839,6 @@ public final class CFMetaData if (cf_def.isSetGc_grace_seconds()) { newCFMD.gcGraceSeconds(cf_def.gc_grace_seconds); } if (cf_def.isSetMin_compaction_threshold()) { newCFMD.minCompactionThreshold(cf_def.min_compaction_threshold); } if (cf_def.isSetMax_compaction_threshold()) { newCFMD.maxCompactionThreshold(cf_def.max_compaction_threshold); } - if (cf_def.isSetKey_alias()) { newCFMD.keyAliases(Collections.singletonList(cf_def.key_alias)); } - if (cf_def.isSetKey_validation_class()) { newCFMD.keyValidator(TypeParser.parse(cf_def.key_validation_class)); } if (cf_def.isSetCompaction_strategy()) newCFMD.compactionStrategyClass = createCompactionStrategy(cf_def.compaction_strategy); if (cf_def.isSetCompaction_strategy_options()) @@ -863,12 +864,18 @@ public final class CFMetaData CompressionParameters cp = CompressionParameters.create(cf_def.compression_options); + if (cf_def.isSetKey_validation_class()) { newCFMD.keyValidator(TypeParser.parse(cf_def.key_validation_class)); } + if (cf_def.isSetKey_alias() && !(newCFMD.keyValidator instanceof CompositeType)) + { + newCFMD.column_metadata.put(cf_def.key_alias, ColumnDefinition.partitionKeyDef(cf_def.key_alias, newCFMD.keyValidator, null)); + } + return newCFMD.comment(cf_def.comment) .replicateOnWrite(cf_def.replicate_on_write) .defaultValidator(TypeParser.parse(cf_def.default_validation_class)) - .keyValidator(TypeParser.parse(cf_def.key_validation_class)) .columnMetadata(ColumnDefinition.fromThrift(cf_def.column_metadata, newCFMD.isSuper())) - .compressionParameters(cp); + .compressionParameters(cp) + .updateCfDef(); } catch (SyntaxException e) { @@ -928,17 +935,6 @@ public final class CFMetaData minCompactionThreshold = cfm.minCompactionThreshold; maxCompactionThreshold = cfm.maxCompactionThreshold; - /* - * Because thrift updates don't know about aliases, we should ignore - * the case where the new aliases are empty. - */ - if (!cfm.keyAliases.isEmpty()) - keyAliases = cfm.keyAliases; - if (!cfm.columnAliases.isEmpty()) - columnAliases = cfm.columnAliases; - if (cfm.valueAlias != null) - valueAlias = cfm.valueAlias; - bloomFilterFpChance = cfm.bloomFilterFpChance; memtableFlushPeriod = cfm.memtableFlushPeriod; caching = cfm.caching; @@ -1084,11 +1080,14 @@ public final class CFMetaData def.setMin_compaction_threshold(minCompactionThreshold); def.setMax_compaction_threshold(maxCompactionThreshold); // We only return the alias if only one is set since thrift don't know about multiple key aliases - if (keyAliases.size() == 1) - def.setKey_alias(keyAliases.get(0)); + if (partitionKeyColumns.size() == 1 && partitionKeyColumns.get(0) != null) + def.setKey_alias(partitionKeyColumns.get(0).name); List column_meta = new ArrayList(column_metadata.size()); for (ColumnDefinition cd : column_metadata.values()) + { + if (cd.type == ColumnDefinition.Type.REGULAR) column_meta.add(cd.toThrift()); + } def.setColumn_metadata(column_meta); def.setCompaction_strategy(compactionStrategyClass.getName()); def.setCompaction_strategy_options(new HashMap(compactionStrategyOptions)); @@ -1231,7 +1230,6 @@ public final class CFMetaData if (cfType == null) throw new ConfigurationException(String.format("Invalid column family type for %s", cfName)); - if (comparator instanceof CounterColumnType) throw new ConfigurationException("CounterColumnType is not a valid comparator"); if (keyValidator instanceof CounterColumnType) @@ -1240,7 +1238,7 @@ public final class CFMetaData // Mixing counter with non counter columns is not supported (#2614) if (defaultValidator instanceof CounterColumnType) { - for (ColumnDefinition def : column_metadata.values()) + for (ColumnDefinition def : regularColumns) if (!(def.getValidator() instanceof CounterColumnType)) throw new ConfigurationException("Cannot add a non counter column (" + getColumnDefinitionComparator(def).getString(def.name) + ") in a counter column family"); } @@ -1251,26 +1249,12 @@ public final class CFMetaData throw new ConfigurationException("Cannot add a counter column (" + getColumnDefinitionComparator(def).getString(def.name) + ") in a non counter column family"); } - // check if any of the columns has name equal to the cf.key_alias - for (ColumnDefinition columndef : column_metadata.values()) - { - for (ByteBuffer alias : keyAliases) - if (alias.equals(columndef.name)) - throw new ConfigurationException("Cannot have key alias equals to a column name: " + UTF8Type.instance.compose(alias)); - - for (ByteBuffer alias : columnAliases) - if (alias.equals(columndef.name)) - throw new ConfigurationException("Cannot have column alias equals to a column name: " + UTF8Type.instance.compose(alias)); - - if (valueAlias != null && valueAlias.equals(columndef.name)) - throw new ConfigurationException("Cannot have value alias equals to a column name: " + UTF8Type.instance.compose(valueAlias)); - } - - for (ByteBuffer alias : keyAliases) - validateAlias(alias, "Key"); - for (ByteBuffer alias : columnAliases) - validateAlias(alias, "Column"); - validateAlias(valueAlias, "Value"); + for (ColumnDefinition def : partitionKeyColumns) + validateAlias(def, "Key"); + for (ColumnDefinition def : clusteringKeyColumns) + validateAlias(def, "Column"); + if (compactValueColumn != null) + validateAlias(compactValueColumn, "Value"); // initialize a set of names NOT in the CF under consideration Set indexNames = existingIndexNames(cfName); @@ -1329,19 +1313,19 @@ public final class CFMetaData for (ColumnFamilyStore cfs : ColumnFamilyStore.all()) { if (cfToExclude == null || !cfs.name.equals(cfToExclude)) - for (ColumnDefinition cd : cfs.metadata.getColumn_metadata().values()) + for (ColumnDefinition cd : cfs.metadata.allColumns()) indexNames.add(cd.getIndexName()); } return indexNames; } - private static void validateAlias(ByteBuffer alias, String msg) throws ConfigurationException + private static void validateAlias(ColumnDefinition alias, String msg) throws ConfigurationException { if (alias != null) { try { - UTF8Type.instance.validate(alias); + UTF8Type.instance.validate(alias.name); } catch (MarshalException e) { @@ -1368,10 +1352,11 @@ public final class CFMetaData * * @param newState The new metadata (for the same CF) * @param modificationTimestamp Timestamp to use for mutation + * @param fromThrift whether the newState comes from thrift * * @return Difference between attributes in form of schema mutation */ - public RowMutation toSchemaUpdate(CFMetaData newState, long modificationTimestamp) + public RowMutation toSchemaUpdate(CFMetaData newState, long modificationTimestamp, boolean fromThrift) { RowMutation rm = new RowMutation(Table.SYSTEM_KS, SystemTable.getSchemaKSKey(ksName)); @@ -1381,7 +1366,14 @@ public final class CFMetaData // columns that are no longer needed for (ColumnDefinition cd : columnDiff.entriesOnlyOnLeft().values()) + { + // Thrift only knows about the REGULAR ColumnDefinition type, so don't consider other type + // are being deleted just because they are not here. + if (fromThrift && cd.type != ColumnDefinition.Type.REGULAR) + continue; + cd.deleteFromSchema(rm, cfName, getColumnDefinitionComparator(cd), modificationTimestamp); + } // newly added columns for (ColumnDefinition cd : columnDiff.entriesOnlyOnRight().values()) @@ -1491,7 +1483,6 @@ public final class CFMetaData cf.addColumn(Column.create(keyValidator.toString(), timestamp, cfName, "key_validator")); cf.addColumn(Column.create(minCompactionThreshold, timestamp, cfName, "min_compaction_threshold")); cf.addColumn(Column.create(maxCompactionThreshold, timestamp, cfName, "max_compaction_threshold")); - cf.addColumn(Column.create(json(aliasesAsStrings(keyAliases)), timestamp, cfName, "key_aliases")); cf.addColumn(bloomFilterFpChance == null ? DeletedColumn.create(ldt, timestamp, cfName, "bloomFilterFpChance") : Column.create(bloomFilterFpChance, timestamp, cfName, "bloom_filter_fp_chance")); cf.addColumn(Column.create(memtableFlushPeriod, timestamp, cfName, "memtable_flush_period_in_ms")); @@ -1499,12 +1490,15 @@ public final class CFMetaData cf.addColumn(Column.create(defaultTimeToLive, timestamp, cfName, "default_time_to_live")); cf.addColumn(Column.create(compactionStrategyClass.getName(), timestamp, cfName, "compaction_strategy_class")); cf.addColumn(Column.create(json(compressionParameters.asThriftOptions()), timestamp, cfName, "compression_parameters")); - cf.addColumn(valueAlias == null ? DeletedColumn.create(ldt, timestamp, cfName, "value_alias") - : Column.create(valueAlias, timestamp, cfName, "value_alias")); - cf.addColumn(Column.create(json(aliasesAsStrings(columnAliases)), timestamp, cfName, "column_aliases")); cf.addColumn(Column.create(json(compactionStrategyOptions), timestamp, cfName, "compaction_strategy_options")); cf.addColumn(Column.create(indexInterval, timestamp, cfName, "index_interval")); cf.addColumn(Column.create(speculativeRetry.toString(), timestamp, cfName, "speculative_retry")); + + // Save the CQL3 metadata "the old way" for compatibility sake + cf.addColumn(Column.create(aliasesToJson(partitionKeyColumns), timestamp, cfName, "key_aliases")); + cf.addColumn(Column.create(aliasesToJson(clusteringKeyColumns), timestamp, cfName, "column_aliases")); + cf.addColumn(compactValueColumn == null ? DeletedColumn.create(ldt, timestamp, cfName, "value_alias") + : Column.create(compactValueColumn.name, timestamp, cfName, "value_alias")); } // Package protected for use by tests @@ -1532,14 +1526,6 @@ public final class CFMetaData if (result.has("comment")) cfm.comment(result.getString("comment")); // We need support the old key_alias for compatibility sake - if (result.has("key_aliases")) - { - cfm.keyAliases(aliasesFromStrings(fromJsonList(result.getString("key_aliases")))); - } - else if (result.has("key_alias")) - { - cfm.keyAliases(Collections.singletonList(result.getBytes("key_alias"))); - } if (result.has("bloom_filter_fp_chance")) cfm.bloomFilterFpChance(result.getDouble("bloom_filter_fp_chance")); if (result.has("memtable_flush_period_in_ms")) @@ -1551,14 +1537,31 @@ public final class CFMetaData cfm.speculativeRetry(SpeculativeRetry.fromString(result.getString("speculative_retry"))); cfm.compactionStrategyClass(createCompactionStrategy(result.getString("compaction_strategy_class"))); cfm.compressionParameters(CompressionParameters.create(fromJsonMap(result.getString("compression_parameters")))); - cfm.columnAliases(aliasesFromStrings(fromJsonList(result.getString("column_aliases")))); - if (result.has("value_alias")) - cfm.valueAlias(result.getBytes("value_alias")); cfm.compactionStrategyOptions(fromJsonMap(result.getString("compaction_strategy_options"))); if (result.has("index_interval")) cfm.indexInterval(result.getInt("index_interval")); if (result.has("populate_io_cache_on_flush")) cfm.populateIoCacheOnFlush(result.getBoolean("populate_io_cache_on_flush")); + + /* + * The info previously hold by key_alias(es), column_alias and value_alias is now stored in column_metadata (because 1) this + * make more sense and 2) this allow to store indexing information). + * However, for upgrade sake we need to still be able to read those old values. Moreover, we cannot easily + * remove those old columns once "converted" to column_metadata because that would screw up nodes that may + * not have upgraded. So for now we keep the both info and in sync, even though its redundant. + * In other words, the ColumnDefinition the following lines add may be replaced later when ColumnDefinition.fromSchema + * is called but that's ok. + */ + if (result.has("key_aliases")) + cfm.addColumnMetadataFromAliases(aliasesFromStrings(fromJsonList(result.getString("key_aliases"))), cfm.keyValidator, ColumnDefinition.Type.PARTITION_KEY); + else if (result.has("key_alias")) + cfm.addColumnMetadataFromAliases(Collections.singletonList(result.getBytes("key_alias")), cfm.keyValidator, ColumnDefinition.Type.PARTITION_KEY); + + cfm.addColumnMetadataFromAliases(aliasesFromStrings(fromJsonList(result.getString("column_aliases"))), cfm.comparator, ColumnDefinition.Type.CLUSTERING_KEY); + + if (result.has("value_alias")) + cfm.addColumnMetadataFromAliases(Collections.singletonList(result.getBytes("value_alias")), cfm.defaultValidator, ColumnDefinition.Type.COMPACT_VALUE); + return cfm; } catch (SyntaxException e) @@ -1571,6 +1574,26 @@ public final class CFMetaData } } + public void addColumnMetadataFromAliases(List aliases, AbstractType comparator, ColumnDefinition.Type type) + { + if (comparator instanceof CompositeType) + { + CompositeType ct = (CompositeType)comparator; + for (int i = 0; i < aliases.size(); ++i) + { + if (aliases.get(i) != null) + column_metadata.put(aliases.get(i), new ColumnDefinition(aliases.get(i), ct.types.get(i), i, type)); + } + } + else + { + assert aliases.size() <= 1; + if (!aliases.isEmpty() && aliases.get(0) != null) + column_metadata.put(aliases.get(0), new ColumnDefinition(aliases.get(0), comparator, null, type)); + } + updateCfDef(); + } + /** * Deserialize CF metadata from low-level representation * @@ -1581,7 +1604,7 @@ public final class CFMetaData CFMetaData cfDef = fromSchemaNoColumns(result); Row serializedColumnDefinitions = ColumnDefinition.readSchema(cfDef.ksName, cfDef.cfName); - return addColumnDefinitionSchema(cfDef, serializedColumnDefinitions).updateCfDef(); + return addColumnDefinitionSchema(cfDef, serializedColumnDefinitions); } private static CFMetaData fromSchema(Row row) @@ -1590,19 +1613,19 @@ public final class CFMetaData return fromSchema(result); } - private List aliasesAsStrings(List rawAliases) + private String aliasesToJson(List rawAliases) { List aliases = new ArrayList(rawAliases.size()); - for (ByteBuffer rawAlias : rawAliases) - aliases.add(UTF8Type.instance.compose(rawAlias)); - return aliases; + for (ColumnDefinition rawAlias : rawAliases) + aliases.add(rawAlias == null ? null : UTF8Type.instance.compose(rawAlias.name)); + return json(aliases); } private static List aliasesFromStrings(List aliases) { List rawAliases = new ArrayList(aliases.size()); for (String alias : aliases) - rawAliases.add(UTF8Type.instance.decompose(alias)); + rawAliases.add(alias == null ? null : UTF8Type.instance.decompose(alias)); return rawAliases; } @@ -1622,27 +1645,36 @@ public final class CFMetaData return rm; } + // The comparator to validate the definition name. + public AbstractType getColumnDefinitionComparator(ColumnDefinition def) { - return getColumnDefinitionComparator(def.componentIndex); + return getComponentComparator(def.componentIndex, def.type); } - public AbstractType getColumnDefinitionComparator(Integer componentIndex) + public AbstractType getComponentComparator(Integer componentIndex, ColumnDefinition.Type type) { - AbstractType cfComparator = cfType == ColumnFamilyType.Super ? ((CompositeType)comparator).types.get(1) : comparator; - if (cfComparator instanceof CompositeType) + switch (type) { - if (componentIndex == null) - return cfComparator; + case REGULAR: + AbstractType cfComparator = cfType == ColumnFamilyType.Super ? ((CompositeType)comparator).types.get(1) : comparator; + if (cfComparator instanceof CompositeType) + { + if (componentIndex == null) + return cfComparator; - List> types = ((CompositeType)cfComparator).types; - AbstractType t = types.get(componentIndex); - assert t != null : "Non-sensical component index"; - return t; - } - else - { - return cfComparator; + List> types = ((CompositeType)cfComparator).types; + AbstractType t = types.get(componentIndex); + assert t != null : "Non-sensical component index"; + return t; + } + else + { + return cfComparator; + } + default: + // CQL3 column names are UTF8 + return UTF8Type.instance; } } @@ -1651,21 +1683,60 @@ public final class CFMetaData { for (ColumnDefinition cd : ColumnDefinition.fromSchema(serializedColumnDefinitions, cfDef)) cfDef.column_metadata.put(cd.name, cd); - return cfDef; + return cfDef.updateCfDef(); + } + + public void addColumnDefinition(ColumnDefinition def) throws ConfigurationException + { + if (column_metadata.containsKey(def.name)) + throw new ConfigurationException(String.format("Cannot add column %s, a column with the same name already exists", getColumnDefinitionComparator(def).getString(def.name))); + + addOrReplaceColumnDefinition(def); } - public void addColumnDefinition(ColumnDefinition def) + // This method doesn't check if a def of the same name already exist and should only be used when we + // know this cannot happen. + public void addOrReplaceColumnDefinition(ColumnDefinition def) { column_metadata.put(def.name, def); + updateCfDef(); } public boolean removeColumnDefinition(ColumnDefinition def) { - return column_metadata.remove(def.name) != null; + boolean removed = column_metadata.remove(def.name) != null; + updateCfDef(); + return removed; + } + + public void renameColumn(ByteBuffer from, String strFrom, ByteBuffer to, String strTo) throws InvalidRequestException + { + ColumnDefinition def = column_metadata.get(from); + if (def == null) + throw new InvalidRequestException(String.format("Cannot rename unknown column %s in table %s", strFrom, cfName)); + + if (column_metadata.get(to) != null) + throw new InvalidRequestException(String.format("Cannot rename column %s to %s in table %s; another column of that name already exist", strFrom, strTo, cfName)); + + if (def.type == ColumnDefinition.Type.REGULAR) + throw new InvalidRequestException(String.format("Cannot rename non PRIMARY KEY part %s", strFrom)); + + ColumnDefinition newDef = def.cloneWithNewName(to); + // don't call addColumnDefinition/removeColumnDefition because we want to avoid recomputing + // the CQL3 cfDef between those two operation + column_metadata.put(newDef.name, newDef); + column_metadata.remove(def.name); } private CFMetaData updateCfDef() { + /* + * TODO: There is definitively some repetition between the CQL3 metadata stored in this + * object (partitionKeyColumns, ...) and the one stored in CFDefinition. + * Ultimately, we should probably merge both. However, there is enough details to fix that + * it's worth doing that in a separate issue. + */ + rebuildCQL3Metadata(); cqlCfDef = new CFDefinition(this); return this; } @@ -1676,21 +1747,160 @@ public final class CFMetaData return cqlCfDef; } + private void rebuildCQL3Metadata() + { + List pkCols = nullInitializedList(keyValidator.componentsCount()); + int nbCkCols = isDense(comparator, column_metadata.values()) + ? comparator.componentsCount() + : comparator.componentsCount() - (hasCollection() ? 2 : 1); + List ckCols = nullInitializedList(nbCkCols); + Set regCols = new HashSet(); + ColumnDefinition compactCol = null; + + for (ColumnDefinition def : column_metadata.values()) + { + switch (def.type) + { + case PARTITION_KEY: + assert !(def.componentIndex == null && keyValidator instanceof CompositeType); + pkCols.set(def.componentIndex == null ? 0 : def.componentIndex, def); + break; + case CLUSTERING_KEY: + assert !(def.componentIndex == null && comparator instanceof CompositeType); + ckCols.set(def.componentIndex == null ? 0 : def.componentIndex, def); + break; + case REGULAR: + regCols.add(def); + break; + case COMPACT_VALUE: + assert compactCol == null : "There shouldn't be more than one compact value defined"; + compactCol = def; + break; + } + } + + // Now actually assign the correct value. This is not atomic, but then again, updating CFMetaData is never atomic anyway. + partitionKeyColumns = pkCols; + clusteringKeyColumns = ckCols; + regularColumns = regCols; + compactValueColumn = compactCol; + } + + private boolean hasCollection() + { + if (isSuper() || !(comparator instanceof CompositeType)) + return false; + + List> types = ((CompositeType)comparator).types; + return types.get(types.size() - 1) instanceof ColumnToCollectionType; + } + + /* + * We call dense a CF for which each component of the comparator is a clustering column, i.e. no + * component is used to store a regular column names. In other words, non-composite static "thrift" + * and CQL3 CF are *not* dense. + * Note that his method is only used by rebuildCQL3Metadata. Once said metadata are built, finding + * if a CF is dense amounts more simply to check if clusteringKeyColumns.size() == comparator.componentsCount(). + */ + private static boolean isDense(AbstractType comparator, Collection defs) + { + /* + * This is a bit subtle to compute because of thrift upgrades. A CQL3 + * CF will have all it's column metadata set up from creation, so + * checking isDense should just be looking the ColumnDefinition of + * type CLUSTERING_KEY having the biggest componentIndex and comparing that + * to comparator.componentsCount. + * However, thrift CF will have no or only some (through ALTER RENAME) + * metadata set and we still need to make our best effort at finding whether + * it is intended as a dense CF or not. + */ + + // First, we compute the number of clustering columns metadata actually defined (and + // whether there is some "hole" in the metadata) + boolean[] definedClusteringKeys = new boolean[comparator.componentsCount()]; + boolean hasRegular = false; + for (ColumnDefinition def : defs) + { + switch (def.type) + { + case CLUSTERING_KEY: + definedClusteringKeys[def.componentIndex == null ? 0 : def.componentIndex] = true; + break; + case REGULAR: + hasRegular = true; + break; + } + } + boolean hasNulls = false; + int maxIdx = -1; + for (int i = definedClusteringKeys.length - 1; i >= 0; i--) + { + if (maxIdx == -1) + { + if (definedClusteringKeys[i]) + maxIdx = i; + } + else + { + if (!definedClusteringKeys[i]) + hasNulls = true; + } + } + + if (comparator instanceof CompositeType) + { + List> types = ((CompositeType)comparator).types; + /* + * There was no real way to define a non-dense composite CF in thrift (the ColumnDefinition.componentIndex + * is not exposed), so consider dense anything that don't look like a CQL3 created CF. + * + * Note that this is not perfect: if someone upgrading from thrift "renames" all but + * the last column alias, the cf will be considered "sparse" and he will be stuck with + * that even though that might not be what he wants. But the simple workaround is + * for that user to rename all the aliases at the same time in the first place. + */ + AbstractType lastType = types.get(types.size() - 1); + if (lastType instanceof ColumnToCollectionType) + return false; + + return !(maxIdx == types.size() - 2 && lastType instanceof UTF8Type && !hasNulls); + } + else + { + /* + * For non-composite, we only need to "detect" case where the CF is clearly used as static. + * For that, just check if we have regular columns metadata sets up and no defined clustering key. + */ + return !(hasRegular && maxIdx == -1); + } + } + + private static List nullInitializedList(int size) + { + List l = new ArrayList(size); + for (int i = 0; i < size; ++i) + l.add(null); + return l; + } + /** - * Returns whether this CFMetaData has information non exposed on thrift so - * that it cannot be correctly handled automatically by thrift clients. + * Returns whether this CFMetaData can be fully translated to a thrift + * definition, i.e. if it doesn't store information that have an equivalent + * in thrift CfDef. */ - public boolean isThriftIncompatible() + public boolean isThriftCompatible() { - if (isSuper() || !cqlCfDef.isComposite) - return false; + // Super CF are always "thrift compatible". But since they may have defs with a componentIndex != null, + // we have to special case here. + if (isSuper()) + return true; - for (ColumnDefinition columnDef : column_metadata.values()) + for (ColumnDefinition def : column_metadata.values()) { - if (columnDef.componentIndex != null) - return true; + if (!def.isThriftCompatible()) + return false; } - return false; + return true; } public void validateColumns(Iterable columns) @@ -1717,9 +1927,6 @@ public final class CFMetaData .append("keyValidator", keyValidator) .append("minCompactionThreshold", minCompactionThreshold) .append("maxCompactionThreshold", maxCompactionThreshold) - .append("keyAliases", keyAliases) - .append("columnAliases", columnAliases) - .append("valueAlias", valueAlias) .append("column_metadata", column_metadata) .append("compactionStrategyClass", compactionStrategyClass) .append("compactionStrategyOptions", compactionStrategyOptions) http://git-wip-us.apache.org/repos/asf/cassandra/blob/a950b925/src/java/org/apache/cassandra/config/ColumnDefinition.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/ColumnDefinition.java b/src/java/org/apache/cassandra/config/ColumnDefinition.java index 81fce0b..fed095d 100644 --- a/src/java/org/apache/cassandra/config/ColumnDefinition.java +++ b/src/java/org/apache/cassandra/config/ColumnDefinition.java @@ -20,6 +20,7 @@ package org.apache.cassandra.config; import java.nio.ByteBuffer; import java.util.*; +import com.google.common.base.Objects; import com.google.common.collect.Maps; import org.apache.cassandra.cql3.QueryProcessor; @@ -37,11 +38,32 @@ import static org.apache.cassandra.utils.FBUtilities.json; public class ColumnDefinition { + /* + * The type of CQL3 column this definition represents. + * There is 3 main type of CQL3 columns: those parts of the partition key, + * those parts of the clustering key and the other, regular ones. + * But when COMPACT STORAGE is used, there is by design only one regular + * column, whose name is not stored in the data contrarily to the column of + * type REGULAR. Hence the COMPACT_VALUE type to distinguish it below. + * + * Note that thrift/CQL2 only know about definitions of type REGULAR (and + * the ones whose componentIndex == null). + */ + + public enum Type + { + PARTITION_KEY, + CLUSTERING_KEY, + REGULAR, + COMPACT_VALUE; + } + public final ByteBuffer name; private AbstractType validator; private IndexType index_type; private Map index_options; private String index_name; + public final Type type; /* * If the column comparator is a composite type, indicates to which @@ -50,7 +72,32 @@ public class ColumnDefinition */ public final Integer componentIndex; - public ColumnDefinition(ByteBuffer name, AbstractType validator, IndexType index_type, Map index_options, String index_name, Integer componentIndex) + public static ColumnDefinition partitionKeyDef(ByteBuffer name, AbstractType validator, Integer componentIndex) + { + return new ColumnDefinition(name, validator, componentIndex, Type.PARTITION_KEY); + } + + public static ColumnDefinition clusteringKeyDef(ByteBuffer name, AbstractType validator, Integer componentIndex) + { + return new ColumnDefinition(name, validator, componentIndex, Type.CLUSTERING_KEY); + } + + public static ColumnDefinition regularDef(ByteBuffer name, AbstractType validator, Integer componentIndex) + { + return new ColumnDefinition(name, validator, componentIndex, Type.REGULAR); + } + + public static ColumnDefinition compactValueDef(ByteBuffer name, AbstractType validator) + { + return new ColumnDefinition(name, validator, null, Type.COMPACT_VALUE); + } + + public ColumnDefinition(ByteBuffer name, AbstractType validator, Integer componentIndex, Type type) + { + this(name, validator, null, null, null, componentIndex, type); + } + + private ColumnDefinition(ByteBuffer name, AbstractType validator, IndexType index_type, Map index_options, String index_name, Integer componentIndex, Type type) { assert name != null && validator != null; this.name = name; @@ -58,11 +105,17 @@ public class ColumnDefinition this.validator = validator; this.componentIndex = componentIndex; this.setIndexType(index_type, index_options); + this.type = type; } public ColumnDefinition clone() { - return new ColumnDefinition(name, validator, index_type, index_options, index_name, componentIndex); + return new ColumnDefinition(name, validator, index_type, index_options, index_name, componentIndex, type); + } + + public ColumnDefinition cloneWithNewName(ByteBuffer newName) + { + return new ColumnDefinition(newName, validator, index_type, index_options, index_name, componentIndex, type); } @Override @@ -74,29 +127,23 @@ public class ColumnDefinition return false; ColumnDefinition that = (ColumnDefinition) o; - if (index_name != null ? !index_name.equals(that.index_name) : that.index_name != null) - return false; - if (index_type != that.index_type) - return false; - if (index_options != null ? !index_options.equals(that.index_options) : that.index_options != null) - return false; - if (!name.equals(that.name)) - return false; - if (componentIndex != null ? !componentIndex.equals(that.componentIndex) : that.componentIndex != null) - return false; - return !(validator != null ? !validator.equals(that.validator) : that.validator != null); + return Objects.equal(name, that.name) + && Objects.equal(validator, that.validator) + && Objects.equal(componentIndex, that.componentIndex) + && Objects.equal(index_name, that.index_name) + && Objects.equal(index_type, that.index_type) + && Objects.equal(index_options, that.index_options); } @Override public int hashCode() { - int result = name != null ? name.hashCode() : 0; - result = 31 * result + (validator != null ? validator.hashCode() : 0); - result = 31 * result + (index_type != null ? index_type.hashCode() : 0); - result = 31 * result + (index_options != null ? index_options.hashCode() : 0); - result = 31 * result + (index_name != null ? index_name.hashCode() : 0); - result = 31 * result + (componentIndex != null ? componentIndex.hashCode() : 0); - return result; + return Objects.hashCode(name, validator, componentIndex, index_name, index_type, index_options); + } + + public boolean isThriftCompatible() + { + return type == ColumnDefinition.Type.REGULAR && componentIndex == null; } public ColumnDef toThrift() @@ -123,7 +170,8 @@ public class ColumnDefinition thriftColumnDef.index_type, thriftColumnDef.index_options, thriftColumnDef.index_name, - isSuper ? 1 : null); + isSuper ? 1 : null, + Type.REGULAR); } public static Map fromThrift(List thriftDefs, boolean isSuper) throws SyntaxException, ConfigurationException @@ -155,6 +203,7 @@ public class ColumnDefinition cf.addColumn(DeletedColumn.create(ldt, timestamp, cfName, comparator.getString(name), "index_options")); cf.addColumn(DeletedColumn.create(ldt, timestamp, cfName, comparator.getString(name), "index_name")); cf.addColumn(DeletedColumn.create(ldt, timestamp, cfName, comparator.getString(name), "component_index")); + cf.addColumn(DeletedColumn.create(ldt, timestamp, cfName, comparator.getString(name), "type")); } public void toSchema(RowMutation rm, String cfName, AbstractType comparator, long timestamp) @@ -171,10 +220,13 @@ public class ColumnDefinition : Column.create(index_name, timestamp, cfName, comparator.getString(name), "index_name")); cf.addColumn(componentIndex == null ? DeletedColumn.create(ldt, timestamp, cfName, comparator.getString(name), "component_index") : Column.create(componentIndex, timestamp, cfName, comparator.getString(name), "component_index")); + cf.addColumn(Column.create(type.toString().toLowerCase(), timestamp, cfName, comparator.getString(name), "type")); } public void apply(ColumnDefinition def, AbstractType comparator) throws ConfigurationException { + assert type == def.type && Objects.equal(componentIndex, def.componentIndex); + if (getIndexType() != null && def.getIndexType() != null) { // If an index is set (and not drop by this update), the validator shouldn't be change to a non-compatible one @@ -186,10 +238,6 @@ public class ColumnDefinition throw new ConfigurationException("Cannot modify index name"); } - if ((componentIndex != null && !componentIndex.equals(def.componentIndex)) - || (componentIndex == null && def.componentIndex != null)) - throw new ConfigurationException(String.format("Cannot modify component index for column %s", comparator.getString(name))); - setValidator(def.getValidator()); setIndexType(def.getIndexType(), def.getIndexOptions()); setIndexName(def.getIndexName()); @@ -228,12 +276,17 @@ public class ColumnDefinition else if (cfm.isSuper()) componentIndex = 1; - cds.add(new ColumnDefinition(cfm.getColumnDefinitionComparator(componentIndex).fromString(result.getString("column_name")), + Type type = result.has("type") + ? Enum.valueOf(Type.class, result.getString("type").toUpperCase()) + : Type.REGULAR; + + cds.add(new ColumnDefinition(cfm.getComponentComparator(componentIndex, type).fromString(result.getString("column_name")), TypeParser.parse(result.getString("validator")), index_type, index_options, index_name, - componentIndex)); + componentIndex, + type)); } catch (RequestValidationException e) { @@ -265,6 +318,7 @@ public class ColumnDefinition ", index_type=" + index_type + ", index_name='" + index_name + '\'' + (componentIndex != null ? ", component_index=" + componentIndex : "") + + ", type=" + type + '}'; } @@ -273,15 +327,27 @@ public class ColumnDefinition return index_name; } - public void setIndexName(String s) + public ColumnDefinition setIndexName(String s) { - index_name = s; + this.index_name = s; + return this; } - public void setIndexType(IndexType index_type, Map index_options) + public ColumnDefinition setIndexType(IndexType index_type, Map index_options) { this.index_type = index_type; this.index_options = index_options; + return this; + } + + public ColumnDefinition setIndex(String s, IndexType index_type, Map index_options) + { + return setIndexName(s).setIndexType(index_type, index_options); + } + + public boolean isIndexed() + { + return index_type != null; } public IndexType getIndexType() @@ -303,18 +369,4 @@ public class ColumnDefinition { this.validator = validator; } - - public static Map getStringMap(Map charMap) - { - if (charMap == null) - return null; - - Map stringMap = new HashMap(); - - for (Map.Entry entry : charMap.entrySet()) - stringMap.put(entry.getKey().toString(), entry.getValue().toString()); - - - return stringMap; - } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/a950b925/src/java/org/apache/cassandra/config/KSMetaData.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/KSMetaData.java b/src/java/org/apache/cassandra/config/KSMetaData.java index 1e58864..e8b9e06 100644 --- a/src/java/org/apache/cassandra/config/KSMetaData.java +++ b/src/java/org/apache/cassandra/config/KSMetaData.java @@ -183,7 +183,7 @@ public final class KSMetaData for (CFMetaData cfm : cfMetaData().values()) { // Don't expose CF that cannot be correctly handle by thrift; see CASSANDRA-4377 for further details - if (!cfm.isThriftIncompatible()) + if (cfm.isThriftCompatible()) cfDefs.add(cfm.toThrift()); } KsDef ksdef = new KsDef(name, strategyClass.getName(), cfDefs); @@ -311,7 +311,11 @@ public final class KSMetaData { Row columnRow = ColumnDefinition.readSchema(cfm.ksName, cfm.cfName); for (ColumnDefinition cd : ColumnDefinition.fromSchema(columnRow, cfm)) - cfm.column_metadata.put(cd.name, cd); + { + // This may replace some existing definition coming from the old key, column and + // value aliases. But that's what we want (see CFMetaData.fromSchemaNoColumns). + cfm.addOrReplaceColumnDefinition(cd); + } } return cfms; http://git-wip-us.apache.org/repos/asf/cassandra/blob/a950b925/src/java/org/apache/cassandra/cql/AlterTableStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql/AlterTableStatement.java b/src/java/org/apache/cassandra/cql/AlterTableStatement.java index a70638e..1f48c1e 100644 --- a/src/java/org/apache/cassandra/cql/AlterTableStatement.java +++ b/src/java/org/apache/cassandra/cql/AlterTableStatement.java @@ -74,22 +74,13 @@ public class AlterTableStatement switch (oType) { case ADD: - if (!cfm.getKeyAliases().isEmpty() && cfm.getKeyAliases().contains(columnName)) - throw new InvalidRequestException("Invalid column name: " - + this.columnName - + ", because it equals to a key alias."); - - cfm.addColumnDefinition(new ColumnDefinition(columnName, - TypeParser.parse(validator), - null, - null, - null, - null)); + cfm.addColumnDefinition(ColumnDefinition.regularDef(columnName, TypeParser.parse(validator), null)); break; case ALTER: // We only look for the first key alias which is ok for CQL2 - if (!cfm.getKeyAliases().isEmpty() && cfm.getKeyAliases().get(0).equals(columnName)) + ColumnDefinition partionKeyDef = cfm.partitionKeyColumns().get(0); + if (partionKeyDef != null && partionKeyDef.name.equals(columnName)) { cfm.keyValidator(TypeParser.parse(validator)); } @@ -97,7 +88,7 @@ public class AlterTableStatement { ColumnDefinition toUpdate = null; - for (ColumnDefinition columnDef : cfm.getColumn_metadata().values()) + for (ColumnDefinition columnDef : cfm.regularColumns()) { if (columnDef.name.equals(columnName)) { @@ -118,7 +109,7 @@ public class AlterTableStatement case DROP: ColumnDefinition toDelete = null; - for (ColumnDefinition columnDef : cfm.getColumn_metadata().values()) + for (ColumnDefinition columnDef : cfm.regularColumns()) { if (columnDef.name.equals(columnName)) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/a950b925/src/java/org/apache/cassandra/cql/CreateColumnFamilyStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql/CreateColumnFamilyStatement.java b/src/java/org/apache/cassandra/cql/CreateColumnFamilyStatement.java index 1dee739..00d8352 100644 --- a/src/java/org/apache/cassandra/cql/CreateColumnFamilyStatement.java +++ b/src/java/org/apache/cassandra/cql/CreateColumnFamilyStatement.java @@ -135,7 +135,7 @@ public class CreateColumnFamilyStatement ? CFPropDefs.comparators.get(col.getValue()) : col.getValue(); AbstractType validator = TypeParser.parse(validatorClassName); - columnDefs.put(columnName, new ColumnDefinition(columnName, validator, null, null, null, null)); + columnDefs.put(columnName, ColumnDefinition.regularDef(columnName, validator, null)); } catch (ConfigurationException e) { @@ -202,7 +202,7 @@ public class CreateColumnFamilyStatement // CQL2 can have null keyAliases if (keyAlias != null) - newCFMD.keyAliases(Collections.singletonList(keyAlias)); + newCFMD.addColumnDefinition(ColumnDefinition.partitionKeyDef(keyAlias, newCFMD.getKeyValidator(), null)); } catch (ConfigurationException e) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/a950b925/src/java/org/apache/cassandra/cql/DropIndexStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql/DropIndexStatement.java b/src/java/org/apache/cassandra/cql/DropIndexStatement.java index 502bdb0..b9f4e5e 100644 --- a/src/java/org/apache/cassandra/cql/DropIndexStatement.java +++ b/src/java/org/apache/cassandra/cql/DropIndexStatement.java @@ -50,7 +50,7 @@ public class DropIndexStatement ColumnDefinition column = findIndexedColumn(cfm); assert column != null; CFMetaData cloned = cfm.clone(); - ColumnDefinition toChange = cloned.getColumn_metadata().get(column.name); + ColumnDefinition toChange = cloned.getColumnDefinition(column.name); assert toChange.getIndexName() != null && toChange.getIndexName().equals(indexName); toChange.setIndexName(null); toChange.setIndexType(null, null); @@ -70,7 +70,7 @@ public class DropIndexStatement private ColumnDefinition findIndexedColumn(CFMetaData cfm) { - for (ColumnDefinition column : cfm.getColumn_metadata().values()) + for (ColumnDefinition column : cfm.regularColumns()) { if (column.getIndexType() != null && column.getIndexName() != null && column.getIndexName().equals(indexName)) return column; http://git-wip-us.apache.org/repos/asf/cassandra/blob/a950b925/src/java/org/apache/cassandra/cql/QueryProcessor.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql/QueryProcessor.java b/src/java/org/apache/cassandra/cql/QueryProcessor.java index b365644..cf2231b 100644 --- a/src/java/org/apache/cassandra/cql/QueryProcessor.java +++ b/src/java/org/apache/cassandra/cql/QueryProcessor.java @@ -650,7 +650,7 @@ public class QueryProcessor ByteBuffer columnName = createIdx.getColumnName().getByteBuffer(); // mutating oldCfm directly would be bad, but mutating a copy is fine. CFMetaData cfm = oldCfm.clone(); - for (ColumnDefinition cd : cfm.getColumn_metadata().values()) + for (ColumnDefinition cd : cfm.regularColumns()) { if (cd.name.equals(columnName)) { @@ -670,7 +670,7 @@ public class QueryProcessor try { cfm.addDefaultIndexNames(); - MigrationManager.announceColumnFamilyUpdate(cfm); + MigrationManager.announceColumnFamilyUpdate(cfm, true); // As far as metadata are concerned, CQL2 == thrift } catch (ConfigurationException e) { @@ -691,7 +691,7 @@ public class QueryProcessor try { CFMetaData updatedCF = dropIdx.generateCFMetadataUpdate(); - MigrationManager.announceColumnFamilyUpdate(updatedCF); + MigrationManager.announceColumnFamilyUpdate(updatedCF, true); // As far as metadata are concerned, CQL2 == thrift } catch (ConfigurationException e) { @@ -748,7 +748,7 @@ public class QueryProcessor try { - MigrationManager.announceColumnFamilyUpdate(alterTable.getCFMetaData(keyspace)); + MigrationManager.announceColumnFamilyUpdate(alterTable.getCFMetaData(keyspace), true); // As far as metadata are concerned, CQL2 == thrift } catch (ConfigurationException e) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/a950b925/src/java/org/apache/cassandra/cql3/CFDefinition.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/CFDefinition.java b/src/java/org/apache/cassandra/cql3/CFDefinition.java index e0e5aef..ebb4e2c 100644 --- a/src/java/org/apache/cassandra/cql3/CFDefinition.java +++ b/src/java/org/apache/cassandra/cql3/CFDefinition.java @@ -65,137 +65,59 @@ public class CFDefinition implements Iterable { this.cfm = cfm; - if (cfm.getKeyValidator() instanceof CompositeType) + this.hasCompositeKey = cfm.getKeyValidator() instanceof CompositeType; + for (int i = 0; i < cfm.partitionKeyColumns().size(); ++i) { - this.hasCompositeKey = true; - CompositeType keyComposite = (CompositeType)cfm.getKeyValidator(); - assert keyComposite.types.size() > 1; - for (int i = 0; i < keyComposite.types.size(); i++) - { - ColumnIdentifier id = getKeyId(cfm, i); - this.keys.put(id, new Name(cfm.ksName, cfm.cfName, id, Name.Kind.KEY_ALIAS, i, keyComposite.types.get(i))); - } + ColumnIdentifier id = getKeyId(cfm, i); + this.keys.put(id, new Name(cfm.ksName, cfm.cfName, id, Name.Kind.KEY_ALIAS, i, cfm.getKeyValidator().getComponents().get(i))); } - else + + this.isComposite = cfm.comparator instanceof CompositeType; + this.hasCollections = cfm.comparator.getComponents().get(cfm.comparator.componentsCount() - 1) instanceof ColumnToCollectionType; + this.isCompact = cfm.clusteringKeyColumns().size() == cfm.comparator.componentsCount(); + for (int i = 0; i < cfm.clusteringKeyColumns().size(); ++i) { - this.hasCompositeKey = false; - ColumnIdentifier id = getKeyId(cfm, 0); - this.keys.put(id, new Name(cfm.ksName, cfm.cfName, id, Name.Kind.KEY_ALIAS, 0, cfm.getKeyValidator())); + ColumnIdentifier id = getColumnId(cfm, i); + this.columns.put(id, new Name(cfm.ksName, cfm.cfName, id, Name.Kind.COLUMN_ALIAS, i, cfm.comparator.getComponents().get(i))); } - if (cfm.comparator instanceof CompositeType) + if (isCompact) { - this.isComposite = true; - CompositeType composite = (CompositeType)cfm.comparator; - /* - * We are a "sparse" composite, i.e. a non-compact one, if either: - * - the last type of the composite is a ColumnToCollectionType - * - or we have one less alias than of composite types and the last type is UTF8Type. - * - some metadata are defined - * - * Note that this is not perfect: if someone upgrading from thrift "renames" all but - * the last column alias, the cf will be considered "sparse" and he will be stuck with - * that even though that might not be what he wants. But the simple workaround is - * for that user to rename all the aliases at the same time in the first place. - */ - int last = composite.types.size() - 1; - AbstractType lastType = composite.types.get(last); - if (!cfm.getColumn_metadata().isEmpty() - || lastType instanceof ColumnToCollectionType - || (cfm.getColumnAliases().size() == last && lastType instanceof UTF8Type)) - { - // "sparse" composite - this.isCompact = false; - this.value = null; - assert cfm.getValueAlias() == null; - // check for collection type - if (lastType instanceof ColumnToCollectionType) - { - --last; - this.hasCollections = true; - } - else - { - this.hasCollections = false; - } - - for (int i = 0; i < last; i++) - { - ColumnIdentifier id = getColumnId(cfm, i); - this.columns.put(id, new Name(cfm.ksName, cfm.cfName, id, Name.Kind.COLUMN_ALIAS, i, composite.types.get(i))); - } - - for (Map.Entry def : cfm.getColumn_metadata().entrySet()) - { - ColumnIdentifier id = new ColumnIdentifier(def.getKey(), cfm.getColumnDefinitionComparator(def.getValue())); - this.metadata.put(id, new Name(cfm.ksName, cfm.cfName, id, Name.Kind.COLUMN_METADATA, def.getValue().getValidator())); - } - } - else - { - // "dense" composite - this.isCompact = true; - this.hasCollections = false; - for (int i = 0; i < composite.types.size(); i++) - { - ColumnIdentifier id = getColumnId(cfm, i); - this.columns.put(id, new Name(cfm.ksName, cfm.cfName, id, Name.Kind.COLUMN_ALIAS, i, composite.types.get(i))); - } - this.value = createValue(cfm); - } + this.value = createValue(cfm); } else { - this.isComposite = false; - this.hasCollections = false; - if (!cfm.getColumnAliases().isEmpty() || cfm.getColumn_metadata().isEmpty()) + this.value = null; + for (ColumnDefinition def : cfm.regularColumns()) { - // dynamic CF - this.isCompact = true; - ColumnIdentifier id = getColumnId(cfm, 0); - Name name = new Name(cfm.ksName, cfm.cfName, id, Name.Kind.COLUMN_ALIAS, 0, cfm.comparator); - this.columns.put(id, name); - this.value = createValue(cfm); - } - else - { - // static CF - this.isCompact = false; - this.value = null; - assert cfm.getValueAlias() == null; - assert cfm.getColumnAliases() == null || cfm.getColumnAliases().isEmpty(); - for (Map.Entry def : cfm.getColumn_metadata().entrySet()) - { - ColumnIdentifier id = new ColumnIdentifier(def.getKey(), cfm.getColumnDefinitionComparator(def.getValue())); - this.metadata.put(id, new Name(cfm.ksName, cfm.cfName, id, Name.Kind.COLUMN_METADATA, def.getValue().getValidator())); - } + ColumnIdentifier id = new ColumnIdentifier(def.name, cfm.getColumnDefinitionComparator(def)); + this.metadata.put(id, new Name(cfm.ksName, cfm.cfName, id, Name.Kind.COLUMN_METADATA, def.getValidator())); } } - assert value == null || metadata.isEmpty(); } private static ColumnIdentifier getKeyId(CFMetaData cfm, int i) { - List definedNames = cfm.getKeyAliases(); + List definedNames = cfm.partitionKeyColumns(); // For compatibility sake, non-composite key default alias is 'key', not 'key1'. - return definedNames == null || i >= definedNames.size() || cfm.getKeyAliases().get(i) == null + return definedNames == null || i >= definedNames.size() || definedNames.get(i) == null ? new ColumnIdentifier(i == 0 ? DEFAULT_KEY_ALIAS : DEFAULT_KEY_ALIAS + (i + 1), false) - : new ColumnIdentifier(cfm.getKeyAliases().get(i), definitionType); + : new ColumnIdentifier(definedNames.get(i).name, definitionType); } private static ColumnIdentifier getColumnId(CFMetaData cfm, int i) { - List definedNames = cfm.getColumnAliases(); - return definedNames == null || i >= definedNames.size() || cfm.getColumnAliases().get(i) == null + List definedNames = cfm.clusteringKeyColumns(); + return definedNames == null || i >= definedNames.size() || definedNames.get(i) == null ? new ColumnIdentifier(DEFAULT_COLUMN_ALIAS + (i + 1), false) - : new ColumnIdentifier(cfm.getColumnAliases().get(i), definitionType); + : new ColumnIdentifier(definedNames.get(i).name, definitionType); } private static ColumnIdentifier getValueId(CFMetaData cfm) { - return cfm.getValueAlias() == null + return cfm.compactValueColumn() == null ? new ColumnIdentifier(DEFAULT_VALUE_ALIAS, false) - : new ColumnIdentifier(cfm.getValueAlias(), definitionType); + : new ColumnIdentifier(cfm.compactValueColumn().name, definitionType); } public ColumnToCollectionType getCollectionType() @@ -367,6 +289,14 @@ public class CFDefinition implements Iterable return columnName == null ? 1 : 0; } + public ByteBuffer get(int i) + { + if (i < 0 || i >= (columnName == null ? 0 : 1)) + throw new IllegalArgumentException(); + + return columnName; + } + public ByteBuffer build() { return columnName == null ? ByteBufferUtil.EMPTY_BYTE_BUFFER : columnName; @@ -383,5 +313,13 @@ public class CFDefinition implements Iterable newBuilder.columnName = columnName; return newBuilder; } + + public ByteBuffer getComponent(int i) + { + if (i != 0 || columnName == null) + throw new IllegalArgumentException(); + + return columnName; + } } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/a950b925/src/java/org/apache/cassandra/cql3/ColumnNameBuilder.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/ColumnNameBuilder.java b/src/java/org/apache/cassandra/cql3/ColumnNameBuilder.java index f7a67de..7f14c7a 100644 --- a/src/java/org/apache/cassandra/cql3/ColumnNameBuilder.java +++ b/src/java/org/apache/cassandra/cql3/ColumnNameBuilder.java @@ -56,6 +56,11 @@ public interface ColumnNameBuilder public int remainingCount(); /** + * @return the ith component in this builder. + */ + public ByteBuffer get(int idx); + + /** * Build the column name. * @return the built column name */ @@ -73,4 +78,14 @@ public interface ColumnNameBuilder * @return the cloned builder. */ public ColumnNameBuilder copy(); + + /** + * Returns the ith component added to this builder. + * + * @param i the component to return + * @return the ith component added to this builder. + * @throws IllegalArgumentException if i >= componentCount(). + */ + public ByteBuffer getComponent(int i); + } http://git-wip-us.apache.org/repos/asf/cassandra/blob/a950b925/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java index 51b9896..ef15691 100644 --- a/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java @@ -112,12 +112,7 @@ public class AlterTableStatement extends SchemaAlteringStatement Integer componentIndex = cfDef.isComposite ? ((CompositeType)meta.comparator).types.size() - (cfDef.hasCollections ? 2 : 1) : null; - cfm.addColumnDefinition(new ColumnDefinition(columnName.key, - type, - null, - null, - null, - componentIndex)); + cfm.addColumnDefinition(ColumnDefinition.regularDef(columnName.key, type, componentIndex)); break; case ALTER: @@ -153,7 +148,6 @@ public class AlterTableStatement extends SchemaAlteringStatement case COLUMN_METADATA: ColumnDefinition column = cfm.getColumnDefinition(columnName.key); column.setValidator(validator.getType()); - cfm.addColumnDefinition(column); break; } break; @@ -171,7 +165,7 @@ public class AlterTableStatement extends SchemaAlteringStatement throw new InvalidRequestException(String.format("Cannot drop PRIMARY KEY part %s", columnName)); case COLUMN_METADATA: ColumnDefinition toDelete = null; - for (ColumnDefinition columnDef : cfm.getColumn_metadata().values()) + for (ColumnDefinition columnDef : cfm.regularColumns()) { if (columnDef.name.equals(columnName.key)) toDelete = columnDef; @@ -191,52 +185,14 @@ public class AlterTableStatement extends SchemaAlteringStatement case RENAME: for (Map.Entry entry : renames.entrySet()) { - CFDefinition.Name from = cfDef.get(entry.getKey()); + ColumnIdentifier from = entry.getKey(); ColumnIdentifier to = entry.getValue(); - if (from == null) - throw new InvalidRequestException(String.format("Column %s was not found in table %s", entry.getKey(), columnFamily())); - - CFDefinition.Name exists = cfDef.get(to); - if (exists != null) - throw new InvalidRequestException(String.format("Cannot rename column %s in table %s to %s; another column of that name already exist", from, columnFamily(), to)); - - switch (from.kind) - { - case KEY_ALIAS: - cfm.keyAliases(rename(from.position, to, cfm.getKeyAliases())); - break; - case COLUMN_ALIAS: - cfm.columnAliases(rename(from.position, to, cfm.getColumnAliases())); - break; - case VALUE_ALIAS: - cfm.valueAlias(to.key); - break; - case COLUMN_METADATA: - throw new InvalidRequestException(String.format("Cannot rename non PRIMARY KEY part %s", from)); - } + cfm.renameColumn(from.key, from.toString(), to.key, to.toString()); } break; } - MigrationManager.announceColumnFamilyUpdate(cfm); - } - - private static List rename(int pos, ColumnIdentifier newName, List aliases) - { - if (pos < aliases.size()) - { - List newList = new ArrayList(aliases); - newList.set(pos, newName.key); - return newList; - } - else - { - List newList = new ArrayList(pos + 1); - for (int i = 0; i < pos; ++i) - newList.add(i < aliases.size() ? aliases.get(i) : null); - newList.add(newName.key); - return newList; - } + MigrationManager.announceColumnFamilyUpdate(cfm, false); } public String toString() http://git-wip-us.apache.org/repos/asf/cassandra/blob/a950b925/src/java/org/apache/cassandra/cql3/statements/CreateColumnFamilyStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateColumnFamilyStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateColumnFamilyStatement.java index f2c3d6a..1648fcb 100644 --- a/src/java/org/apache/cassandra/cql3/statements/CreateColumnFamilyStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/CreateColumnFamilyStatement.java @@ -93,7 +93,7 @@ public class CreateColumnFamilyStatement extends SchemaAlteringStatement for (Map.Entry col : columns.entrySet()) { - columnDefs.put(col.getKey().key, new ColumnDefinition(col.getKey().key, col.getValue(), null, null, null, componentIndex)); + columnDefs.put(col.getKey().key, ColumnDefinition.regularDef(col.getKey().key, col.getValue(), componentIndex)); } return columnDefs; @@ -131,11 +131,13 @@ public class CreateColumnFamilyStatement extends SchemaAlteringStatement public void applyPropertiesTo(CFMetaData cfmd) throws RequestValidationException { cfmd.defaultValidator(defaultValidator) - .columnMetadata(getColumns()) .keyValidator(keyValidator) - .keyAliases(keyAliases) - .columnAliases(columnAliases) - .valueAlias(valueAlias); + .columnMetadata(getColumns()); + + cfmd.addColumnMetadataFromAliases(keyAliases, keyValidator, ColumnDefinition.Type.PARTITION_KEY); + cfmd.addColumnMetadataFromAliases(columnAliases, comparator, ColumnDefinition.Type.CLUSTERING_KEY); + if (valueAlias != null) + cfmd.addColumnMetadataFromAliases(Collections.singletonList(valueAlias), defaultValidator, ColumnDefinition.Type.COMPACT_VALUE); properties.applyToCFMetadata(cfmd); }