cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From slebre...@apache.org
Subject [08/51] [partial] cassandra git commit: Storage engine refactor, a.k.a CASSANDRA-8099
Date Tue, 30 Jun 2015 10:47:32 GMT
http://git-wip-us.apache.org/repos/asf/cassandra/blob/a991b648/src/java/org/apache/cassandra/thrift/ThriftConversion.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/thrift/ThriftConversion.java b/src/java/org/apache/cassandra/thrift/ThriftConversion.java
index adb925e..148fd68 100644
--- a/src/java/org/apache/cassandra/thrift/ThriftConversion.java
+++ b/src/java/org/apache/cassandra/thrift/ThriftConversion.java
@@ -23,31 +23,35 @@ import java.util.*;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
 import com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.cache.CachingOptions;
 import org.apache.cassandra.config.*;
 import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.cql3.UntypedResultSet;
-import org.apache.cassandra.db.ColumnFamilyType;
-import org.apache.cassandra.schema.LegacySchemaTables;
-import org.apache.cassandra.db.WriteType;
-import org.apache.cassandra.db.composites.CellNameType;
-import org.apache.cassandra.db.composites.CellNames;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.filter.RowFilter;
 import org.apache.cassandra.db.marshal.*;
 import org.apache.cassandra.exceptions.*;
 import org.apache.cassandra.io.compress.CompressionParameters;
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
 import org.apache.cassandra.locator.LocalStrategy;
 import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.schema.LegacySchemaTables;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.UUIDGen;
 
 /**
  * Static utility methods to convert internal structure to and from thrift ones.
+ *
+ * 
  */
 public class ThriftConversion
 {
+    private static final Logger logger = LoggerFactory.getLogger(ThriftConversion.class);
+
     public static org.apache.cassandra.db.ConsistencyLevel fromThrift(ConsistencyLevel cl)
     {
         switch (cl)
@@ -136,21 +140,14 @@ public class ThriftConversion
         return new TimedOutException();
     }
 
-    public static List<org.apache.cassandra.db.IndexExpression> indexExpressionsFromThrift(List<IndexExpression> exprs)
+    public static RowFilter rowFilterFromThrift(CFMetaData metadata, List<IndexExpression> exprs)
     {
-        if (exprs == null)
-            return null;
-
-        if (exprs.isEmpty())
-            return Collections.emptyList();
+        if (exprs == null || exprs.isEmpty())
+            return RowFilter.NONE;
 
-        List<org.apache.cassandra.db.IndexExpression> converted = new ArrayList<>(exprs.size());
+        RowFilter converted = RowFilter.forThrift(exprs.size());
         for (IndexExpression expr : exprs)
-        {
-            converted.add(new org.apache.cassandra.db.IndexExpression(expr.column_name,
-                                                                      Operator.valueOf(expr.op.name()),
-                                                                      expr.value));
-        }
+            converted.addThriftExpression(metadata, expr.column_name, Operator.valueOf(expr.op.name()), expr.value);
         return converted;
     }
 
@@ -184,50 +181,75 @@ public class ThriftConversion
     public static CFMetaData fromThrift(CfDef cf_def)
     throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException
     {
-        return internalFromThrift(cf_def, Collections.<ColumnDefinition>emptyList());
+        // This is a creation: the table is dense if it doesn't define any column_metadata
+        boolean isDense = cf_def.column_metadata == null || cf_def.column_metadata.isEmpty();
+        return internalFromThrift(cf_def, true, Collections.<ColumnDefinition>emptyList(), isDense);
     }
 
     public static CFMetaData fromThriftForUpdate(CfDef cf_def, CFMetaData toUpdate)
     throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException
     {
-        return internalFromThrift(cf_def, toUpdate.allColumns());
+        return internalFromThrift(cf_def, false, toUpdate.allColumns(), toUpdate.isDense());
     }
 
-    // Convert a thrift CfDef, given a list of ColumnDefinitions to copy over to the created CFMetadata before the CQL metadata are rebuild
-    private static CFMetaData internalFromThrift(CfDef cf_def, Collection<ColumnDefinition> previousCQLMetadata)
-    throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException
+    private static boolean isSuper(String thriftColumnType)
+    throws org.apache.cassandra.exceptions.InvalidRequestException
     {
-        ColumnFamilyType cfType = ColumnFamilyType.create(cf_def.column_type);
-        if (cfType == null)
-            throw new org.apache.cassandra.exceptions.InvalidRequestException("Invalid column type " + cf_def.column_type);
+        switch (thriftColumnType.toLowerCase())
+        {
+            case "standard": return false;
+            case "super": return true;
+            default: throw new org.apache.cassandra.exceptions.InvalidRequestException("Invalid column type " + thriftColumnType);
+        }
+    }
 
+    /**
+     * Convert a thrift CfDef.
+     * <p>,
+     * This is used both for creation and update of CF.
+     *
+     * @param cf_def the thrift CfDef to convert.
+     * @param isCreation whether that is a new table creation or not.
+     * @param previousCQLMetadata if it is not a table creation, the previous
+     * definitions of the tables (which we use to preserve the CQL metadata).
+     * If it is a table creation, this will be empty.
+     * @param isDense whether the table is dense or not.
+     *
+     * @return the converted table definition.
+     */
+    private static CFMetaData internalFromThrift(CfDef cf_def,
+                                                 boolean isCreation,
+                                                 Collection<ColumnDefinition> previousCQLMetadata,
+                                                 boolean isDense)
+    throws org.apache.cassandra.exceptions.InvalidRequestException, ConfigurationException
+    {
         applyImplicitDefaults(cf_def);
 
         try
         {
+            boolean isSuper = isSuper(cf_def.column_type);
             AbstractType<?> rawComparator = TypeParser.parse(cf_def.comparator_type);
-            AbstractType<?> subComparator = cfType == ColumnFamilyType.Standard
-                    ? null
-                    : cf_def.subcomparator_type == null ? BytesType.instance : TypeParser.parse(cf_def.subcomparator_type);
+            AbstractType<?> subComparator = isSuper
+                                          ? cf_def.subcomparator_type == null ? BytesType.instance : TypeParser.parse(cf_def.subcomparator_type)
+                                          : null;
 
-            AbstractType<?> fullRawComparator = CFMetaData.makeRawAbstractType(rawComparator, subComparator);
+            AbstractType<?> keyValidator = cf_def.isSetKey_validation_class() ? TypeParser.parse(cf_def.key_validation_class) : BytesType.instance;
+            AbstractType<?> defaultValidator = TypeParser.parse(cf_def.default_validation_class);
 
-            AbstractType<?> keyValidator = cf_def.isSetKey_validation_class() ? TypeParser.parse(cf_def.key_validation_class) : null;
-
-            // Convert the REGULAR definitions from the input CfDef
+            // Convert the definitions from the input CfDef
             List<ColumnDefinition> defs = fromThrift(cf_def.keyspace, cf_def.name, rawComparator, subComparator, cf_def.column_metadata);
 
-            // Add the keyAlias if there is one, since that's on CQL metadata that thrift can actually change (for
+            // Add the keyAlias if there is one, since that's a CQL metadata that thrift can actually change (for
             // historical reasons)
             boolean hasKeyAlias = cf_def.isSetKey_alias() && keyValidator != null && !(keyValidator instanceof CompositeType);
             if (hasKeyAlias)
-                defs.add(ColumnDefinition.partitionKeyDef(cf_def.keyspace, cf_def.name, cf_def.key_alias, keyValidator, null));
+                defs.add(ColumnDefinition.partitionKeyDef(cf_def.keyspace, cf_def.name, UTF8Type.instance.getString(cf_def.key_alias), keyValidator, null));
 
             // Now add any CQL metadata that we want to copy, skipping the keyAlias if there was one
             for (ColumnDefinition def : previousCQLMetadata)
             {
                 // isPartOfCellName basically means 'is not just a CQL metadata'
-                if (def.isPartOfCellName())
+                if (def.isPartOfCellName(false, isSuper))
                     continue;
 
                 if (def.kind == ColumnDefinition.Kind.PARTITION_KEY && hasKeyAlias)
@@ -236,18 +258,25 @@ public class ThriftConversion
                 defs.add(def);
             }
 
-            CellNameType comparator = CellNames.fromAbstractType(fullRawComparator, CFMetaData.calculateIsDense(fullRawComparator, defs));
-
             UUID cfId = Schema.instance.getId(cf_def.keyspace, cf_def.name);
             if (cfId == null)
                 cfId = UUIDGen.getTimeUUID();
 
-            CFMetaData newCFMD = new CFMetaData(cf_def.keyspace, cf_def.name, cfType, comparator, cfId);
+            boolean isCompound = isSuper ? false : (rawComparator instanceof CompositeType);
+            boolean isCounter = defaultValidator instanceof CounterColumnType;
 
-            newCFMD.addAllColumnDefinitions(defs);
+            // If it's a thrift table creation, adds the default CQL metadata for the new table
+            if (isCreation)
+                addDefaultCQLMetadata(defs,
+                                      cf_def.keyspace,
+                                      cf_def.name,
+                                      hasKeyAlias ? null : keyValidator,
+                                      rawComparator,
+                                      subComparator,
+                                      defaultValidator);
+
+            CFMetaData newCFMD = CFMetaData.create(cf_def.keyspace, cf_def.name, cfId, isDense, isCompound, isSuper, isCounter, defs);
 
-            if (keyValidator != null)
-                newCFMD.keyValidator(keyValidator);
             if (cf_def.isSetGc_grace_seconds())
                 newCFMD.gcGraceSeconds(cf_def.gc_grace_seconds);
             if (cf_def.isSetMin_compaction_threshold())
@@ -280,9 +309,7 @@ public class ThriftConversion
                 newCFMD.triggers(triggerDefinitionsFromThrift(cf_def.triggers));
 
             return newCFMD.comment(cf_def.comment)
-                          .defaultValidator(TypeParser.parse(cf_def.default_validation_class))
-                          .compressionParameters(CompressionParameters.create(cf_def.compression_options))
-                          .rebuild();
+                          .compressionParameters(CompressionParameters.create(cf_def.compression_options));
         }
         catch (SyntaxException | MarshalException e)
         {
@@ -290,6 +317,48 @@ public class ThriftConversion
         }
     }
 
+    private static void addDefaultCQLMetadata(Collection<ColumnDefinition> defs,
+                                              String ks,
+                                              String cf,
+                                              AbstractType<?> keyValidator,
+                                              AbstractType<?> comparator,
+                                              AbstractType<?> subComparator,
+                                              AbstractType<?> defaultValidator)
+    {
+        CompactTables.DefaultNames names = CompactTables.defaultNameGenerator(defs);
+        if (keyValidator != null)
+        {
+            if (keyValidator instanceof CompositeType)
+            {
+                List<AbstractType<?>> subTypes = ((CompositeType)keyValidator).types;
+                for (int i = 0; i < subTypes.size(); i++)
+                    defs.add(ColumnDefinition.partitionKeyDef(ks, cf, names.defaultPartitionKeyName(), subTypes.get(i), i));
+            }
+            else
+            {
+                defs.add(ColumnDefinition.partitionKeyDef(ks, cf, names.defaultPartitionKeyName(), keyValidator, null));
+            }
+        }
+
+        if (subComparator != null)
+        {
+            // SuperColumn tables: we use a special map to hold dynamic values within a given super column
+            defs.add(ColumnDefinition.clusteringKeyDef(ks, cf, names.defaultClusteringName(), comparator, 0));
+            defs.add(ColumnDefinition.regularDef(ks, cf, CompactTables.SUPER_COLUMN_MAP_COLUMN_STR, MapType.getInstance(subComparator, defaultValidator, true), null));
+        }
+        else
+        {
+            List<AbstractType<?>> subTypes = comparator instanceof CompositeType
+                                           ? ((CompositeType)comparator).types
+                                           : Collections.<AbstractType<?>>singletonList(comparator);
+
+            for (int i = 0; i < subTypes.size(); i++)
+                defs.add(ColumnDefinition.clusteringKeyDef(ks, cf, names.defaultClusteringName(), subTypes.get(i), i));
+
+            defs.add(ColumnDefinition.regularDef(ks, cf, names.defaultCompactValueName(), defaultValidator, null));
+        }
+    }
+
     /** applies implicit defaults to cf definition. useful in updates */
     private static void applyImplicitDefaults(org.apache.cassandra.thrift.CfDef cf_def)
     {
@@ -355,30 +424,30 @@ public class ThriftConversion
     public static CfDef toThrift(CFMetaData cfm)
     {
         CfDef def = new CfDef(cfm.ksName, cfm.cfName);
-        def.setColumn_type(cfm.cfType.name());
+        def.setColumn_type(cfm.isSuper() ? "Super" : "Standard");
 
         if (cfm.isSuper())
         {
             def.setComparator_type(cfm.comparator.subtype(0).toString());
-            def.setSubcomparator_type(cfm.comparator.subtype(1).toString());
+            def.setSubcomparator_type(cfm.thriftColumnNameType().toString());
         }
         else
         {
-            def.setComparator_type(cfm.comparator.toString());
+            def.setComparator_type(LegacyLayout.makeLegacyComparator(cfm).toString());
         }
 
         def.setComment(Strings.nullToEmpty(cfm.getComment()));
         def.setRead_repair_chance(cfm.getReadRepairChance());
         def.setDclocal_read_repair_chance(cfm.getDcLocalReadRepairChance());
         def.setGc_grace_seconds(cfm.getGcGraceSeconds());
-        def.setDefault_validation_class(cfm.getDefaultValidator().toString());
+        def.setDefault_validation_class(cfm.makeLegacyDefaultValidator().toString());
         def.setKey_validation_class(cfm.getKeyValidator().toString());
         def.setMin_compaction_threshold(cfm.getMinCompactionThreshold());
         def.setMax_compaction_threshold(cfm.getMaxCompactionThreshold());
         // We only return the alias if only one is set since thrift don't know about multiple key aliases
         if (cfm.partitionKeyColumns().size() == 1)
             def.setKey_alias(cfm.partitionKeyColumns().get(0).name.bytes);
-        def.setColumn_metadata(columnDefinitionsToThrift(cfm.allColumns()));
+        def.setColumn_metadata(columnDefinitionsToThrift(cfm, cfm.allColumns()));
         def.setCompaction_strategy(cfm.compactionStrategyClass.getName());
         def.setCompaction_strategy_options(new HashMap<>(cfm.compactionStrategyOptions));
         def.setCompression_options(cfm.compressionParameters.asThriftOptions());
@@ -402,8 +471,9 @@ public class ThriftConversion
                                               ColumnDef thriftColumnDef)
     throws SyntaxException, ConfigurationException
     {
+        boolean isSuper = thriftSubcomparator != null;
         // For super columns, the componentIndex is 1 because the ColumnDefinition applies to the column component.
-        Integer componentIndex = thriftSubcomparator != null ? 1 : null;
+        Integer componentIndex = isSuper ? 1 : null;
         AbstractType<?> comparator = thriftSubcomparator == null ? thriftComparator : thriftSubcomparator;
         try
         {
@@ -414,15 +484,18 @@ public class ThriftConversion
             throw new ConfigurationException(String.format("Column name %s is not valid for comparator %s", ByteBufferUtil.bytesToHex(thriftColumnDef.name), comparator));
         }
 
+        // In our generic layout, we store thrift defined columns as static, but this doesn't work for super columns so we
+        // use a regular definition (and "dynamic" columns are handled in a map).
+        ColumnDefinition.Kind kind = isSuper ? ColumnDefinition.Kind.REGULAR : ColumnDefinition.Kind.STATIC;
         return new ColumnDefinition(ksName,
                                     cfName,
-                                    new ColumnIdentifier(ByteBufferUtil.clone(thriftColumnDef.name), comparator),
+                                    ColumnIdentifier.getInterned(ByteBufferUtil.clone(thriftColumnDef.name), comparator),
                                     TypeParser.parse(thriftColumnDef.validation_class),
                                     thriftColumnDef.index_type == null ? null : org.apache.cassandra.config.IndexType.valueOf(thriftColumnDef.index_type.name()),
                                     thriftColumnDef.index_options,
                                     thriftColumnDef.index_name,
                                     componentIndex,
-                                    ColumnDefinition.Kind.REGULAR);
+                                    kind);
     }
 
     private static List<ColumnDefinition> fromThrift(String ksName,
@@ -456,11 +529,11 @@ public class ThriftConversion
         return cd;
     }
 
-    private static List<ColumnDef> columnDefinitionsToThrift(Collection<ColumnDefinition> columns)
+    private static List<ColumnDef> columnDefinitionsToThrift(CFMetaData metadata, Collection<ColumnDefinition> columns)
     {
         List<ColumnDef> thriftDefs = new ArrayList<>(columns.size());
         for (ColumnDefinition def : columns)
-            if (def.kind == ColumnDefinition.Kind.REGULAR)
+            if (def.isPartOfCellName(metadata.isCQLTable(), metadata.isSuper()))
                 thriftDefs.add(ThriftConversion.toThrift(def));
         return thriftDefs;
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/a991b648/src/java/org/apache/cassandra/thrift/ThriftResultsMerger.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/thrift/ThriftResultsMerger.java b/src/java/org/apache/cassandra/thrift/ThriftResultsMerger.java
new file mode 100644
index 0000000..ccb6e74
--- /dev/null
+++ b/src/java/org/apache/cassandra/thrift/ThriftResultsMerger.java
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.thrift;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.rows.*;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.MapType;
+import org.apache.cassandra.db.partitions.*;
+
+/**
+ * Given an iterator on a partition of a compact table, this return an iterator that merges the
+ * static row columns with the other results.
+ *
+ * Compact tables stores thrift column_metadata as static columns (see CompactTables for
+ * details). When reading for thrift however, we want to merge those static values with other
+ * results because:
+ *   1) on thrift, all "columns" are sorted together, whether or not they are declared
+ *      column_metadata.
+ *   2) it's possible that a table add a value for a "dynamic" column, and later that column
+ *      is statically defined. Merging "static" and "dynamic" columns make sure we don't miss
+ *      a value prior to the column declaration.
+ *
+ * For example, if a thrift table declare 2 columns "c1" and "c5" and the results from a query
+ * is:
+ *    Partition: static: { c1: 3, c5: 4 }
+ *                 "a" : { value : 2 }
+ *                 "c3": { value : 8 }
+ *                 "c7": { value : 1 }
+ * then this class transform it into:
+ *    Partition:   "a" : { value : 2 }
+ *                 "c1": { value : 3 }
+ *                 "c3": { value : 8 }
+ *                 "c5": { value : 4 }
+ *                 "c7": { value : 1 }
+ */
+public class ThriftResultsMerger extends WrappingUnfilteredPartitionIterator
+{
+    private final int nowInSec;
+
+    private ThriftResultsMerger(UnfilteredPartitionIterator wrapped, int nowInSec)
+    {
+        super(wrapped);
+        this.nowInSec = nowInSec;
+    }
+
+    public static UnfilteredPartitionIterator maybeWrap(UnfilteredPartitionIterator iterator, CFMetaData metadata, int nowInSec)
+    {
+        if (!metadata.isStaticCompactTable() && !metadata.isSuper())
+            return iterator;
+
+        return new ThriftResultsMerger(iterator, nowInSec);
+    }
+
+    public static UnfilteredRowIterator maybeWrap(UnfilteredRowIterator iterator, int nowInSec)
+    {
+        if (!iterator.metadata().isStaticCompactTable() && !iterator.metadata().isSuper())
+            return iterator;
+
+        return iterator.metadata().isSuper()
+             ? new SuperColumnsPartitionMerger(iterator, nowInSec)
+             : new PartitionMerger(iterator, nowInSec);
+    }
+
+    protected UnfilteredRowIterator computeNext(UnfilteredRowIterator iter)
+    {
+        return iter.metadata().isSuper()
+             ? new SuperColumnsPartitionMerger(iter, nowInSec)
+             : new PartitionMerger(iter, nowInSec);
+    }
+
+    private static class PartitionMerger extends WrappingUnfilteredRowIterator
+    {
+        private final int nowInSec;
+
+        // We initialize lazily to avoid having this iterator fetch the wrapped iterator before it's actually asked for it.
+        private boolean isInit;
+
+        private Row staticRow;
+        private int i; // the index of the next column of static row to return
+
+        private ReusableRow nextToMerge;
+        private Unfiltered nextFromWrapped;
+
+        private PartitionMerger(UnfilteredRowIterator results, int nowInSec)
+        {
+            super(results);
+            assert results.metadata().isStaticCompactTable();
+            this.nowInSec = nowInSec;
+        }
+
+        private void init()
+        {
+            assert !isInit;
+            this.staticRow = super.staticRow();
+            assert staticRow.columns().complexColumnCount() == 0;
+
+            this.nextToMerge = createReusableRow();
+            updateNextToMerge();
+            isInit = true;
+        }
+
+        @Override
+        public Row staticRow()
+        {
+            return Rows.EMPTY_STATIC_ROW;
+        }
+
+        private ReusableRow createReusableRow()
+        {
+            return new ReusableRow(metadata().clusteringColumns().size(), metadata().partitionColumns().regulars, true, metadata().isCounter());
+        }
+
+        @Override
+        public boolean hasNext()
+        {
+            if (!isInit)
+                init();
+
+            return nextFromWrapped != null || nextToMerge != null || super.hasNext();
+        }
+
+        @Override
+        public Unfiltered next()
+        {
+            if (!isInit)
+                init();
+
+            if (nextFromWrapped == null && super.hasNext())
+                nextFromWrapped = super.next();
+
+            if (nextFromWrapped == null)
+            {
+                if (nextToMerge == null)
+                    throw new NoSuchElementException();
+
+                return consumeNextToMerge();
+            }
+
+            if (nextToMerge == null)
+                return consumeNextWrapped();
+
+            int cmp = metadata().comparator.compare(nextToMerge, nextFromWrapped);
+            if (cmp < 0)
+                return consumeNextToMerge();
+            if (cmp > 0)
+                return consumeNextWrapped();
+
+            // Same row, but we know the row has only a single column so just pick the more recent
+            assert nextFromWrapped instanceof Row;
+            ReusableRow row = createReusableRow();
+            Rows.merge((Row)consumeNextWrapped(), consumeNextToMerge(), columns().regulars, row.writer(), nowInSec);
+            return row;
+        }
+
+        private Unfiltered consumeNextWrapped()
+        {
+            Unfiltered toReturn = nextFromWrapped;
+            nextFromWrapped = null;
+            return toReturn;
+        }
+
+        private Row consumeNextToMerge()
+        {
+            Row toReturn = nextToMerge;
+            updateNextToMerge();
+            return toReturn;
+        }
+
+        private void updateNextToMerge()
+        {
+            while (i < staticRow.columns().simpleColumnCount())
+            {
+                Cell cell = staticRow.getCell(staticRow.columns().getSimple(i++));
+                if (cell != null)
+                {
+                    // Given a static cell, the equivalent row uses the column name as clustering and the
+                    // value as unique cell value.
+                    Row.Writer writer = nextToMerge.writer();
+                    writer.writeClusteringValue(cell.column().name.bytes);
+                    writer.writeCell(metadata().compactValueColumn(), cell.isCounterCell(), cell.value(), cell.livenessInfo(), cell.path());
+                    writer.endOfRow();
+                    return;
+                }
+            }
+            // Nothing more to merge.
+            nextToMerge = null;
+        }
+    }
+
+    private static class SuperColumnsPartitionMerger extends WrappingUnfilteredRowIterator
+    {
+        private final int nowInSec;
+        private final ReusableRow reusableRow;
+        private final ColumnDefinition superColumnMapColumn;
+        private final AbstractType<?> columnComparator;
+
+        private SuperColumnsPartitionMerger(UnfilteredRowIterator results, int nowInSec)
+        {
+            super(results);
+            assert results.metadata().isSuper();
+            this.nowInSec = nowInSec;
+
+            this.superColumnMapColumn = results.metadata().compactValueColumn();
+            assert superColumnMapColumn != null && superColumnMapColumn.type instanceof MapType;
+
+            this.reusableRow = new ReusableRow(results.metadata().clusteringColumns().size(),
+                                               Columns.of(superColumnMapColumn),
+                                               true,
+                                               results.metadata().isCounter());
+            this.columnComparator = ((MapType)superColumnMapColumn.type).nameComparator();
+        }
+
+        @Override
+        public Unfiltered next()
+        {
+            Unfiltered next = super.next();
+            if (next.kind() != Unfiltered.Kind.ROW)
+                return next;
+
+            Row row = (Row)next;
+            Row.Writer writer = reusableRow.writer();
+            row.clustering().writeTo(writer);
+
+            PeekingIterator<Cell> staticCells = Iterators.peekingIterator(makeStaticCellIterator(row));
+            if (!staticCells.hasNext())
+                return row;
+
+            Iterator<Cell> cells = row.getCells(superColumnMapColumn);
+            PeekingIterator<Cell> dynamicCells = Iterators.peekingIterator(cells.hasNext() ? cells : Collections.<Cell>emptyIterator());
+
+            while (staticCells.hasNext() && dynamicCells.hasNext())
+            {
+                Cell staticCell = staticCells.peek();
+                Cell dynamicCell = dynamicCells.peek();
+                int cmp = columnComparator.compare(staticCell.column().name.bytes, dynamicCell.path().get(0));
+                if (cmp < 0)
+                {
+                    staticCell = staticCells.next();
+                    writer.writeCell(superColumnMapColumn, staticCell.isCounterCell(), staticCell.value(), staticCell.livenessInfo(), CellPath.create(staticCell.column().name.bytes));
+                }
+                else if (cmp > 0)
+                {
+                    dynamicCells.next().writeTo(writer);
+                }
+                else
+                {
+                    staticCell = staticCells.next();
+                    Cell toMerge = Cells.create(superColumnMapColumn,
+                                                 staticCell.isCounterCell(),
+                                                 staticCell.value(),
+                                                 staticCell.livenessInfo(),
+                                                 CellPath.create(staticCell.column().name.bytes));
+                    Cells.reconcile(toMerge, dynamicCells.next(), nowInSec).writeTo(writer);
+                }
+            }
+
+            while (staticCells.hasNext())
+            {
+                Cell staticCell = staticCells.next();
+                writer.writeCell(superColumnMapColumn, staticCell.isCounterCell(), staticCell.value(), staticCell.livenessInfo(), CellPath.create(staticCell.column().name.bytes));
+            }
+            while (dynamicCells.hasNext())
+            {
+                dynamicCells.next().writeTo(writer);
+            }
+
+            writer.endOfRow();
+            return reusableRow;
+        }
+
+        private static Iterator<Cell> makeStaticCellIterator(final Row row)
+        {
+            return new AbstractIterator<Cell>()
+            {
+                private int i;
+
+                protected Cell computeNext()
+                {
+                    while (i < row.columns().simpleColumnCount())
+                    {
+                        Cell cell = row.getCell(row.columns().getSimple(i++));
+                        if (cell != null)
+                            return cell;
+                    }
+                    return endOfData();
+                }
+            };
+        }
+    }
+}
+

http://git-wip-us.apache.org/repos/asf/cassandra/blob/a991b648/src/java/org/apache/cassandra/thrift/ThriftValidation.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/thrift/ThriftValidation.java b/src/java/org/apache/cassandra/thrift/ThriftValidation.java
index 3768952..dd5bf98 100644
--- a/src/java/org/apache/cassandra/thrift/ThriftValidation.java
+++ b/src/java/org/apache/cassandra/thrift/ThriftValidation.java
@@ -24,17 +24,10 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.*;
-import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.Attributes;
 import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.composites.*;
-import org.apache.cassandra.db.filter.IDiskAtomFilter;
-import org.apache.cassandra.db.filter.NamesQueryFilter;
-import org.apache.cassandra.db.filter.SliceQueryFilter;
-import org.apache.cassandra.db.index.SecondaryIndex;
 import org.apache.cassandra.db.index.SecondaryIndexManager;
 import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.ColumnToCollectionType;
-import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.serializers.MarshalException;
@@ -123,7 +116,7 @@ public class ThriftValidation
      */
     public static void validateColumnPath(CFMetaData metadata, ColumnPath column_path) throws org.apache.cassandra.exceptions.InvalidRequestException
     {
-        if (metadata.cfType == ColumnFamilyType.Standard)
+        if (!metadata.isSuper())
         {
             if (column_path.super_column != null)
             {
@@ -151,7 +144,7 @@ public class ThriftValidation
 
     public static void validateColumnParent(CFMetaData metadata, ColumnParent column_parent) throws org.apache.cassandra.exceptions.InvalidRequestException
     {
-        if (metadata.cfType == ColumnFamilyType.Standard)
+        if (!metadata.isSuper())
         {
             if (column_parent.super_column != null)
             {
@@ -168,19 +161,19 @@ public class ThriftValidation
     // column_path_or_parent is a ColumnPath for remove, where the "column" is optional even for a standard CF
     static void validateColumnPathOrParent(CFMetaData metadata, ColumnPath column_path_or_parent) throws org.apache.cassandra.exceptions.InvalidRequestException
     {
-        if (metadata.cfType == ColumnFamilyType.Standard)
+        if (metadata.isSuper())
         {
-            if (column_path_or_parent.super_column != null)
+            if (column_path_or_parent.super_column == null && column_path_or_parent.column != null)
             {
-                throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn may not be specified for standard CF " + metadata.cfName);
+                throw new org.apache.cassandra.exceptions.InvalidRequestException("A column cannot be specified without specifying a super column for removal on super CF "
+                                                                          + metadata.cfName);
             }
         }
-        if (metadata.cfType == ColumnFamilyType.Super)
+        else
         {
-            if (column_path_or_parent.super_column == null && column_path_or_parent.column != null)
+            if (column_path_or_parent.super_column != null)
             {
-                throw new org.apache.cassandra.exceptions.InvalidRequestException("A column cannot be specified without specifying a super column for removal on super CF "
-                                                                          + metadata.cfName);
+                throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn may not be specified for standard CF " + metadata.cfName);
             }
         }
         if (column_path_or_parent.column != null)
@@ -193,13 +186,30 @@ public class ThriftValidation
         }
     }
 
+    private static AbstractType<?> getThriftColumnNameComparator(CFMetaData metadata, ByteBuffer superColumnName)
+    {
+        if (!metadata.isSuper())
+            return LegacyLayout.makeLegacyComparator(metadata);
+
+        if (superColumnName == null)
+        {
+            // comparator for super column name
+            return metadata.comparator.subtype(0);
+        }
+        else
+        {
+            // comparator for sub columns
+            return metadata.thriftColumnNameType();
+        }
+    }
+
     /**
      * Validates the column names but not the parent path or data
      */
     private static void validateColumnNames(CFMetaData metadata, ByteBuffer superColumnName, Iterable<ByteBuffer> column_names)
     throws org.apache.cassandra.exceptions.InvalidRequestException
     {
-        int maxNameLength = Cell.MAX_NAME_LENGTH;
+        int maxNameLength = LegacyLayout.MAX_CELL_NAME_LENGTH;
 
         if (superColumnName != null)
         {
@@ -207,10 +217,10 @@ public class ThriftValidation
                 throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn name length must not be greater than " + maxNameLength);
             if (superColumnName.remaining() == 0)
                 throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn name must not be empty");
-            if (metadata.cfType == ColumnFamilyType.Standard)
+            if (!metadata.isSuper())
                 throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn specified to table " + metadata.cfName + " containing normal columns");
         }
-        AbstractType<?> comparator = SuperColumns.getComparatorFor(metadata, superColumnName);
+        AbstractType<?> comparator = getThriftColumnNameComparator(metadata, superColumnName);
         boolean isCQL3Table = !metadata.isThriftCompatible();
         for (ByteBuffer name : column_names)
         {
@@ -229,31 +239,26 @@ public class ThriftValidation
 
             if (isCQL3Table)
             {
-                // CQL3 table don't support having only part of their composite column names set
-                Composite composite = metadata.comparator.fromByteBuffer(name);
-
-                int minComponents = metadata.comparator.clusteringPrefixSize() + 1;
-                if (composite.size() < minComponents)
-                    throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Not enough components (found %d but %d expected) for column name since %s is a CQL3 table",
-                                                                                                    composite.size(), minComponents, metadata.cfName));
-
-                // Furthermore, the column name must be a declared one.
-                int columnIndex = metadata.comparator.clusteringPrefixSize();
-                ByteBuffer CQL3ColumnName = composite.get(columnIndex);
-                if (!CQL3ColumnName.hasRemaining())
-                    continue; // Row marker, ok
-
-                ColumnIdentifier columnId = new ColumnIdentifier(CQL3ColumnName, metadata.comparator.subtype(columnIndex));
-                if (metadata.getColumnDefinition(columnId) == null)
-                    throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Invalid cell for CQL3 table %s. The CQL3 column component (%s) does not correspond to a defined CQL3 column",
-                                                                                                    metadata.cfName, columnId));
-
-                // On top of that, if we have a collection component, he (CQL3) column must be a collection
-                if (metadata.comparator.hasCollections() && composite.size() == metadata.comparator.size())
+                try
+                {
+                    LegacyLayout.LegacyCellName cname = LegacyLayout.decodeCellName(metadata, name);
+                    assert cname.clustering.size() == metadata.comparator.size();
+
+                    // CQL3 table don't support having only part of their composite column names set
+                    for (int i = 0; i < cname.clustering.size(); i++)
+                    {
+                        if (cname.clustering.get(i) == null)
+                            throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Not enough components (found %d but %d expected) for column name since %s is a CQL3 table",
+                                                                                                            i, metadata.comparator.size() + 1, metadata.cfName));
+                    }
+
+                    // On top of that, if we have a collection component, the (CQL3) column must be a collection
+                    if (cname.column != null && cname.collectionElement != null && !cname.column.type.isCollection())
+                        throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Invalid collection component, %s is not a collection", cname.column.name));
+                }
+                catch (IllegalArgumentException | UnknownColumnException e)
                 {
-                    ColumnToCollectionType collectionType = metadata.comparator.collectionType();
-                    if (!collectionType.defined.containsKey(CQL3ColumnName))
-                        throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Invalid collection component, %s is not a collection", UTF8Type.instance.getString(CQL3ColumnName)));
+                    throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Error validating cell name for CQL3 table %s: %s", metadata.cfName, e.getMessage()));
                 }
             }
         }
@@ -269,13 +274,13 @@ public class ThriftValidation
         if (range.count < 0)
             throw new org.apache.cassandra.exceptions.InvalidRequestException("get_slice requires non-negative count");
 
-        int maxNameLength = Cell.MAX_NAME_LENGTH;
+        int maxNameLength = LegacyLayout.MAX_CELL_NAME_LENGTH;
         if (range.start.remaining() > maxNameLength)
             throw new org.apache.cassandra.exceptions.InvalidRequestException("range start length cannot be larger than " + maxNameLength);
         if (range.finish.remaining() > maxNameLength)
             throw new org.apache.cassandra.exceptions.InvalidRequestException("range finish length cannot be larger than " + maxNameLength);
 
-        AbstractType<?> comparator = SuperColumns.getComparatorFor(metadata, column_parent.super_column);
+        AbstractType<?> comparator = getThriftColumnNameComparator(metadata, column_parent.super_column);
         try
         {
             comparator.validate(range.start);
@@ -295,7 +300,7 @@ public class ThriftValidation
         }
     }
 
-    public static void validateColumnOrSuperColumn(CFMetaData metadata, ByteBuffer key, ColumnOrSuperColumn cosc)
+    public static void validateColumnOrSuperColumn(CFMetaData metadata, ColumnOrSuperColumn cosc)
             throws org.apache.cassandra.exceptions.InvalidRequestException
     {
         boolean isCommutative = metadata.isCounter();
@@ -316,7 +321,7 @@ public class ThriftValidation
 
             validateTtl(cosc.column);
             validateColumnPath(metadata, new ColumnPath(metadata.cfName).setSuper_column((ByteBuffer)null).setColumn(cosc.column.name));
-            validateColumnData(metadata, key, null, cosc.column);
+            validateColumnData(metadata, null, cosc.column);
         }
 
         if (cosc.super_column != null)
@@ -327,7 +332,7 @@ public class ThriftValidation
             for (Column c : cosc.super_column.columns)
             {
                 validateColumnPath(metadata, new ColumnPath(metadata.cfName).setSuper_column(cosc.super_column.name).setColumn(c.name));
-                validateColumnData(metadata, key, cosc.super_column.name, c);
+                validateColumnData(metadata, cosc.super_column.name, c);
             }
         }
 
@@ -356,8 +361,8 @@ public class ThriftValidation
             if (column.ttl <= 0)
                 throw new org.apache.cassandra.exceptions.InvalidRequestException("ttl must be positive");
 
-            if (column.ttl > ExpiringCell.MAX_TTL)
-                throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("ttl is too large. requested (%d) maximum (%d)", column.ttl, ExpiringCell.MAX_TTL));
+            if (column.ttl > Attributes.MAX_TTL)
+                throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("ttl is too large. requested (%d) maximum (%d)", column.ttl, Attributes.MAX_TTL));
         }
         else
         {
@@ -366,7 +371,7 @@ public class ThriftValidation
         }
     }
 
-    public static void validateMutation(CFMetaData metadata, ByteBuffer key, Mutation mut)
+    public static void validateMutation(CFMetaData metadata, Mutation mut)
             throws org.apache.cassandra.exceptions.InvalidRequestException
     {
         ColumnOrSuperColumn cosc = mut.column_or_supercolumn;
@@ -383,7 +388,7 @@ public class ThriftValidation
 
         if (cosc != null)
         {
-            validateColumnOrSuperColumn(metadata, key, cosc);
+            validateColumnOrSuperColumn(metadata, cosc);
         }
         else
         {
@@ -400,7 +405,7 @@ public class ThriftValidation
         if (del.predicate != null)
             validateSlicePredicate(metadata, del.super_column, del.predicate);
 
-        if (metadata.cfType == ColumnFamilyType.Standard && del.super_column != null)
+        if (!metadata.isSuper() && del.super_column != null)
         {
             String msg = String.format("Deletion of super columns is not possible on a standard table (KeySpace=%s Table=%s Deletion=%s)", metadata.ksName, metadata.cfName, del);
             throw new org.apache.cassandra.exceptions.InvalidRequestException(msg);
@@ -409,7 +414,7 @@ public class ThriftValidation
         if (metadata.isCounter())
         {
             // forcing server timestamp even if a timestamp was set for coherence with other counter operation
-            del.timestamp = System.currentTimeMillis();
+            del.timestamp = FBUtilities.timestampMicros();
         }
         else if (!del.isSetTimestamp())
         {
@@ -432,7 +437,7 @@ public class ThriftValidation
     /**
      * Validates the data part of the column (everything in the column object but the name, which is assumed to be valid)
      */
-    public static void validateColumnData(CFMetaData metadata, ByteBuffer key, ByteBuffer scName, Column column) throws org.apache.cassandra.exceptions.InvalidRequestException
+    public static void validateColumnData(CFMetaData metadata, ByteBuffer scName, Column column) throws org.apache.cassandra.exceptions.InvalidRequestException
     {
         validateTtl(column);
         if (!column.isSetValue())
@@ -440,14 +445,17 @@ public class ThriftValidation
         if (!column.isSetTimestamp())
             throw new org.apache.cassandra.exceptions.InvalidRequestException("Column timestamp is required");
 
-        CellName cn = scName == null
-                    ? metadata.comparator.cellFromByteBuffer(column.name)
-                    : metadata.comparator.makeCellName(scName, column.name);
         try
         {
-            AbstractType<?> validator = metadata.getValueValidator(cn);
-            if (validator != null)
-                validator.validate(column.value);
+            LegacyLayout.LegacyCellName cn = LegacyLayout.decodeCellName(metadata, scName, column.name);
+            cn.column.validateCellValue(column.value);
+
+            // Indexed column values cannot be larger than 64K.  See CASSANDRA-3057/4240 for more details
+            Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager.validate(cn.column, column.value, null);
+        }
+        catch (UnknownColumnException e)
+        {
+            throw new org.apache.cassandra.exceptions.InvalidRequestException(e.getMessage());
         }
         catch (MarshalException me)
         {
@@ -458,25 +466,9 @@ public class ThriftValidation
                                                                       me.getMessage(),
                                                                       metadata.ksName,
                                                                       metadata.cfName,
-                                                                      (SuperColumns.getComparatorFor(metadata, scName != null)).getString(column.name)));
+                                                                      (getThriftColumnNameComparator(metadata, scName)).getString(column.name)));
         }
 
-        // Indexed column values cannot be larger than 64K.  See CASSANDRA-3057/4240 for more details
-        SecondaryIndex failedIndex = Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager.validate(key, asDBColumn(cn, column));
-        if (failedIndex != null)
-                    throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Can't index column value of size %d for index %s in CF %s of KS %s",
-                                                                              column.value.remaining(),
-                                                                              failedIndex.getIndexName(),
-                                                                              metadata.cfName,
-                                                                              metadata.ksName));
-    }
-
-    private static Cell asDBColumn(CellName name, Column column)
-    {
-        if (column.ttl <= 0)
-            return new BufferCell(name, column.value, column.timestamp);
-        else
-            return new BufferExpiringCell(name, column.value, column.timestamp, column.ttl);
     }
 
     /**
@@ -535,8 +527,8 @@ public class ThriftValidation
         else if (range.start_key != null && range.end_token != null)
         {
             // start_token/end_token can wrap, but key/token should not
-            RowPosition stop = p.getTokenFactory().fromString(range.end_token).maxKeyBound();
-            if (RowPosition.ForKey.get(range.start_key, p).compareTo(stop) > 0 && !stop.isMinimum())
+            PartitionPosition stop = p.getTokenFactory().fromString(range.end_token).maxKeyBound();
+            if (PartitionPosition.ForKey.get(range.start_key, p).compareTo(stop) > 0 && !stop.isMinimum())
                 throw new org.apache.cassandra.exceptions.InvalidRequestException("Start key's token sorts after end token");
         }
 
@@ -577,7 +569,7 @@ public class ThriftValidation
             return false;
 
         SecondaryIndexManager idxManager = Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager;
-        AbstractType<?> nameValidator = SuperColumns.getComparatorFor(metadata, null);
+        AbstractType<?> nameValidator = getThriftColumnNameComparator(metadata, null);
 
         boolean isIndexed = false;
         for (IndexExpression expression : index_clause)
@@ -597,11 +589,18 @@ public class ThriftValidation
             if (expression.value.remaining() > 0xFFFF)
                 throw new org.apache.cassandra.exceptions.InvalidRequestException("Index expression values may not be larger than 64K");
 
-            CellName name = metadata.comparator.cellFromByteBuffer(expression.column_name);
-            AbstractType<?> valueValidator = metadata.getValueValidator(name);
+            ColumnDefinition def = metadata.getColumnDefinition(expression.column_name);
+            if (def == null)
+            {
+                if (!metadata.isCompactTable())
+                    throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Unknown column %s", nameValidator.getString(expression.column_name)));
+
+                def = metadata.compactValueColumn();
+            }
+
             try
             {
-                valueValidator.validate(expression.value);
+                def.type.validate(expression.value);
             }
             catch (MarshalException me)
             {
@@ -611,7 +610,7 @@ public class ThriftValidation
                                                                                   me.getMessage()));
             }
 
-            isIndexed |= (expression.op == IndexOperator.EQ) && idxManager.indexes(name);
+            isIndexed |= (expression.op == IndexOperator.EQ) && idxManager.indexes(def);
         }
 
         return isIndexed;
@@ -637,32 +636,32 @@ public class ThriftValidation
             throw new org.apache.cassandra.exceptions.InvalidRequestException("system keyspace is not user-modifiable");
     }
 
-    public static IDiskAtomFilter asIFilter(SlicePredicate sp, CFMetaData metadata, ByteBuffer superColumn)
-    {
-        SliceRange sr = sp.slice_range;
-        IDiskAtomFilter filter;
-
-        CellNameType comparator = metadata.isSuper()
-                                ? new SimpleDenseCellNameType(metadata.comparator.subtype(superColumn == null ? 0 : 1))
-                                : metadata.comparator;
-        if (sr == null)
-        {
-
-            SortedSet<CellName> ss = new TreeSet<CellName>(comparator);
-            for (ByteBuffer bb : sp.column_names)
-                ss.add(comparator.cellFromByteBuffer(bb));
-            filter = new NamesQueryFilter(ss);
-        }
-        else
-        {
-            filter = new SliceQueryFilter(comparator.fromByteBuffer(sr.start),
-                                          comparator.fromByteBuffer(sr.finish),
-                                          sr.reversed,
-                                          sr.count);
-        }
-
-        if (metadata.isSuper())
-            filter = SuperColumns.fromSCFilter(metadata.comparator, superColumn, filter);
-        return filter;
-    }
+    //public static IDiskAtomFilter asIFilter(SlicePredicate sp, CFMetaData metadata, ByteBuffer superColumn)
+    //{
+    //    SliceRange sr = sp.slice_range;
+    //    IDiskAtomFilter filter;
+
+    //    CellNameType comparator = metadata.isSuper()
+    //                            ? new SimpleDenseCellNameType(metadata.comparator.subtype(superColumn == null ? 0 : 1))
+    //                            : metadata.comparator;
+    //    if (sr == null)
+    //    {
+
+    //        SortedSet<CellName> ss = new TreeSet<CellName>(comparator);
+    //        for (ByteBuffer bb : sp.column_names)
+    //            ss.add(comparator.cellFromByteBuffer(bb));
+    //        filter = new NamesQueryFilter(ss);
+    //    }
+    //    else
+    //    {
+    //        filter = new SliceQueryFilter(comparator.fromByteBuffer(sr.start),
+    //                                      comparator.fromByteBuffer(sr.finish),
+    //                                      sr.reversed,
+    //                                      sr.count);
+    //    }
+
+    //    if (metadata.isSuper())
+    //        filter = SuperColumns.fromSCFilter(metadata.comparator, superColumn, filter);
+    //    return filter;
+    //}
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/a991b648/src/java/org/apache/cassandra/tools/SSTableExport.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableExport.java b/src/java/org/apache/cassandra/tools/SSTableExport.java
deleted file mode 100644
index 9f833e7..0000000
--- a/src/java/org/apache/cassandra/tools/SSTableExport.java
+++ /dev/null
@@ -1,480 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.tools;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.*;
-
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.commons.cli.*;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.composites.CellNameType;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.io.sstable.*;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.codehaus.jackson.JsonGenerator;
-import org.codehaus.jackson.map.ObjectMapper;
-
-/**
- * Export SSTables to JSON format.
- */
-public class SSTableExport
-{
-    private static final ObjectMapper jsonMapper = new ObjectMapper();
-
-    private static final String KEY_OPTION = "k";
-    private static final String EXCLUDEKEY_OPTION = "x";
-    private static final String ENUMERATEKEYS_OPTION = "e";
-
-    private static final Options options = new Options();
-    private static CommandLine cmd;
-
-    static
-    {
-        Option optKey = new Option(KEY_OPTION, true, "Row key");
-        // Number of times -k <key> can be passed on the command line.
-        optKey.setArgs(500);
-        options.addOption(optKey);
-
-        Option excludeKey = new Option(EXCLUDEKEY_OPTION, true, "Excluded row key");
-        // Number of times -x <key> can be passed on the command line.
-        excludeKey.setArgs(500);
-        options.addOption(excludeKey);
-
-        Option optEnumerate = new Option(ENUMERATEKEYS_OPTION, false, "enumerate keys only");
-        options.addOption(optEnumerate);
-
-        // disabling auto close of the stream
-        jsonMapper.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false);
-    }
-
-    /**
-     * Checks if PrintStream error and throw exception
-     *
-     * @param out The PrintStream to be check
-     */
-    private static void checkStream(PrintStream out) throws IOException
-    {
-        if (out.checkError())
-            throw new IOException("Error writing output stream");
-    }
-
-    /**
-     * JSON Hash Key serializer
-     *
-     * @param out   The output steam to write data
-     * @param value value to set as a key
-     */
-    private static void writeKey(PrintStream out, String value)
-    {
-        writeJSON(out, value);
-        out.print(": ");
-    }
-
-    private static List<Object> serializeAtom(OnDiskAtom atom, CFMetaData cfMetaData)
-    {
-        if (atom instanceof Cell)
-        {
-            return serializeColumn((Cell) atom, cfMetaData);
-        }
-        else
-        {
-            assert atom instanceof RangeTombstone;
-            RangeTombstone rt = (RangeTombstone) atom;
-            ArrayList<Object> serializedColumn = new ArrayList<Object>();
-            serializedColumn.add(cfMetaData.comparator.getString(rt.min));
-            serializedColumn.add(cfMetaData.comparator.getString(rt.max));
-            serializedColumn.add(rt.data.markedForDeleteAt);
-            serializedColumn.add("t");
-            serializedColumn.add(rt.data.localDeletionTime);
-            return serializedColumn;
-        }
-    }
-
-    /**
-     * Serialize a given cell to a List of Objects that jsonMapper knows how to turn into strings.  Type is
-     *
-     * human_readable_name, value, timestamp, [flag, [options]]
-     *
-     * Value is normally the human readable value as rendered by the validator, but for deleted cells we
-     * give the local deletion time instead.
-     *
-     * Flag may be exactly one of {d,e,c} for deleted, expiring, or counter:
-     *  - No options for deleted cells
-     *  - If expiring, options will include the TTL and local deletion time.
-     *  - If counter, options will include timestamp of last delete
-     *
-     * @param cell     cell presentation
-     * @param cfMetaData Column Family metadata (to get validator)
-     * @return cell as serialized list
-     */
-    private static List<Object> serializeColumn(Cell cell, CFMetaData cfMetaData)
-    {
-        CellNameType comparator = cfMetaData.comparator;
-        ArrayList<Object> serializedColumn = new ArrayList<Object>();
-
-        serializedColumn.add(comparator.getString(cell.name()));
-
-        if (cell instanceof DeletedCell)
-        {
-            serializedColumn.add(cell.getLocalDeletionTime());
-        }
-        else
-        {
-            AbstractType<?> validator = cfMetaData.getValueValidator(cell.name());
-            serializedColumn.add(validator.getString(cell.value()));
-        }
-
-        serializedColumn.add(cell.timestamp());
-
-        if (cell instanceof DeletedCell)
-        {
-            serializedColumn.add("d");
-        }
-        else if (cell instanceof ExpiringCell)
-        {
-            serializedColumn.add("e");
-            serializedColumn.add(((ExpiringCell) cell).getTimeToLive());
-            serializedColumn.add(cell.getLocalDeletionTime());
-        }
-        else if (cell instanceof CounterCell)
-        {
-            serializedColumn.add("c");
-            serializedColumn.add(((CounterCell) cell).timestampOfLastDelete());
-        }
-
-        return serializedColumn;
-    }
-
-    /**
-     * Get portion of the columns and serialize in loop while not more columns left in the row
-     *
-     * @param row SSTableIdentityIterator row representation with Column Family
-     * @param key Decorated Key for the required row
-     * @param out output stream
-     */
-    private static void serializeRow(SSTableIdentityIterator row, DecoratedKey key, PrintStream out)
-    {
-        serializeRow(row.getColumnFamily().deletionInfo(), row, row.getColumnFamily().metadata(), key, out);
-    }
-
-    private static void serializeRow(DeletionInfo deletionInfo, Iterator<OnDiskAtom> atoms, CFMetaData metadata, DecoratedKey key, PrintStream out)
-    {
-        out.print("{");
-        writeKey(out, "key");
-        writeJSON(out, metadata.getKeyValidator().getString(key.getKey()));
-        out.print(",\n");
-
-        if (!deletionInfo.isLive())
-        {
-            out.print(" ");
-            writeKey(out, "metadata");
-            out.print("{");
-            writeKey(out, "deletionInfo");
-            writeJSON(out, deletionInfo.getTopLevelDeletion());
-            out.print("}");
-            out.print(",\n");
-        }
-
-        out.print(" ");
-        writeKey(out, "cells");
-        out.print("[");
-        while (atoms.hasNext())
-        {
-            writeJSON(out, serializeAtom(atoms.next(), metadata));
-
-            if (atoms.hasNext())
-                out.print(",\n           ");
-        }
-        out.print("]");
-
-        out.print("}");
-    }
-
-    /**
-     * Enumerate row keys from an SSTableReader and write the result to a PrintStream.
-     *
-     * @param desc the descriptor of the file to export the rows from
-     * @param outs PrintStream to write the output to
-     * @param metadata Metadata to print keys in a proper format
-     * @throws IOException on failure to read/write input/output
-     */
-    public static void enumeratekeys(Descriptor desc, PrintStream outs, CFMetaData metadata)
-    throws IOException
-    {
-        try (KeyIterator iter = new KeyIterator(desc))
-        {
-            DecoratedKey lastKey = null;
-            while (iter.hasNext())
-            {
-                DecoratedKey key = iter.next();
-
-                // validate order of the keys in the sstable
-                if (lastKey != null && lastKey.compareTo(key) > 0)
-                    throw new IOException("Key out of order! " + lastKey + " > " + key);
-                lastKey = key;
-
-                outs.println(metadata.getKeyValidator().getString(key.getKey()));
-                checkStream(outs); // flushes
-            }
-        }
-    }
-
-    /**
-     * Export specific rows from an SSTable and write the resulting JSON to a PrintStream.
-     *
-     * @param desc     the descriptor of the sstable to read from
-     * @param outs     PrintStream to write the output to
-     * @param toExport the keys corresponding to the rows to export
-     * @param excludes keys to exclude from export
-     * @param metadata Metadata to print keys in a proper format
-     * @throws IOException on failure to read/write input/output
-     */
-    public static void export(Descriptor desc, PrintStream outs, Collection<String> toExport, String[] excludes, CFMetaData metadata) throws IOException
-    {
-        SSTableReader sstable = SSTableReader.open(desc);
-
-        try (RandomAccessReader dfile = sstable.openDataReader())
-        {
-            IPartitioner partitioner = sstable.partitioner;
-
-            if (excludes != null)
-                toExport.removeAll(Arrays.asList(excludes));
-
-            outs.println("[");
-
-            int i = 0;
-
-            // last key to compare order
-            DecoratedKey lastKey = null;
-
-            for (String key : toExport)
-            {
-                DecoratedKey decoratedKey = partitioner.decorateKey(metadata.getKeyValidator().fromString(key));
-
-                if (lastKey != null && lastKey.compareTo(decoratedKey) > 0)
-                    throw new IOException("Key out of order! " + lastKey + " > " + decoratedKey);
-
-                lastKey = decoratedKey;
-
-                RowIndexEntry entry = sstable.getPosition(decoratedKey, SSTableReader.Operator.EQ);
-                if (entry == null)
-                    continue;
-
-                dfile.seek(entry.position);
-                ByteBufferUtil.readWithShortLength(dfile); // row key
-                DeletionInfo deletionInfo = new DeletionInfo(DeletionTime.serializer.deserialize(dfile));
-
-                Iterator<OnDiskAtom> atomIterator = sstable.metadata.getOnDiskIterator(dfile, sstable.descriptor.version);
-                checkStream(outs);
-
-                if (i != 0)
-                    outs.println(",");
-                i++;
-                serializeRow(deletionInfo, atomIterator, sstable.metadata, decoratedKey, outs);
-            }
-
-            outs.println("\n]");
-            outs.flush();
-        }
-    }
-
-    // This is necessary to accommodate the test suite since you cannot open a Reader more
-    // than once from within the same process.
-    static void export(SSTableReader reader, PrintStream outs, String[] excludes) throws IOException
-    {
-        Set<String> excludeSet = new HashSet<String>();
-
-        if (excludes != null)
-            excludeSet = new HashSet<>(Arrays.asList(excludes));
-
-        SSTableIdentityIterator row;
-        ISSTableScanner scanner = reader.getScanner();
-        try
-        {
-            outs.println("[");
-
-            int i = 0;
-
-            // collecting keys to export
-            while (scanner.hasNext())
-            {
-                row = (SSTableIdentityIterator) scanner.next();
-
-                String currentKey = row.getColumnFamily().metadata().getKeyValidator().getString(row.getKey().getKey());
-
-                if (excludeSet.contains(currentKey))
-                    continue;
-                else if (i != 0)
-                    outs.println(",");
-
-                serializeRow(row, row.getKey(), outs);
-                checkStream(outs);
-
-                i++;
-            }
-
-            outs.println("\n]");
-            outs.flush();
-        }
-        finally
-        {
-            scanner.close();
-        }
-    }
-
-    /**
-     * Export an SSTable and write the resulting JSON to a PrintStream.
-     *
-     * @param desc     the descriptor of the sstable to read from
-     * @param outs     PrintStream to write the output to
-     * @param excludes keys to exclude from export
-     * @throws IOException on failure to read/write input/output
-     */
-    public static void export(Descriptor desc, PrintStream outs, String[] excludes) throws IOException
-    {
-        export(SSTableReader.open(desc), outs, excludes);
-    }
-
-    /**
-     * Export an SSTable and write the resulting JSON to standard out.
-     *
-     * @param desc     the descriptor of the sstable to read from
-     * @param excludes keys to exclude from export
-     * @throws IOException on failure to read/write SSTable/standard out
-     */
-    public static void export(Descriptor desc, String[] excludes) throws IOException
-    {
-        export(desc, System.out, excludes);
-    }
-
-    /**
-     * Given arguments specifying an SSTable, and optionally an output file,
-     * export the contents of the SSTable to JSON.
-     *
-     * @param args command lines arguments
-     * @throws ConfigurationException on configuration failure (wrong params given)
-     */
-    public static void main(String[] args) throws ConfigurationException
-    {
-        System.err.println("WARNING: please note that sstable2json is now deprecated and will be removed in Cassandra 3.0. "
-                         + "Please see https://issues.apache.org/jira/browse/CASSANDRA-9618 for details.");
-
-        String usage = String.format("Usage: %s <sstable> [-k key [-k key [...]] -x key [-x key [...]]]%n", SSTableExport.class.getName());
-
-        CommandLineParser parser = new PosixParser();
-        try
-        {
-            cmd = parser.parse(options, args);
-        }
-        catch (ParseException e1)
-        {
-            System.err.println(e1.getMessage());
-            System.err.println(usage);
-            System.exit(1);
-        }
-
-
-        if (cmd.getArgs().length != 1)
-        {
-            System.err.println("You must supply exactly one sstable");
-            System.err.println(usage);
-            System.exit(1);
-        }
-
-
-        String[] keys = cmd.getOptionValues(KEY_OPTION);
-        String[] excludes = cmd.getOptionValues(EXCLUDEKEY_OPTION);
-        String ssTableFileName = new File(cmd.getArgs()[0]).getAbsolutePath();
-
-        Schema.instance.loadFromDisk(false);
-        Descriptor descriptor = Descriptor.fromFilename(ssTableFileName);
-
-        // Start by validating keyspace name
-        if (Schema.instance.getKSMetaData(descriptor.ksname) == null)
-        {
-            System.err.println(String.format("Filename %s references to nonexistent keyspace: %s!",
-                                             ssTableFileName, descriptor.ksname));
-            System.exit(1);
-        }
-        Keyspace keyspace = Keyspace.open(descriptor.ksname);
-
-        // Make it works for indexes too - find parent cf if necessary
-        String baseName = descriptor.cfname;
-        if (descriptor.cfname.contains("."))
-        {
-            String[] parts = descriptor.cfname.split("\\.", 2);
-            baseName = parts[0];
-        }
-
-        // IllegalArgumentException will be thrown here if ks/cf pair does not exist
-        ColumnFamilyStore cfStore = null;
-        try
-        {
-            cfStore = keyspace.getColumnFamilyStore(baseName);
-        }
-        catch (IllegalArgumentException e)
-        {
-            System.err.println(String.format("The provided table is not part of this cassandra keyspace: keyspace = %s, table = %s",
-                                             descriptor.ksname, descriptor.cfname));
-            System.exit(1);
-        }
-
-        try
-        {
-            if (cmd.hasOption(ENUMERATEKEYS_OPTION))
-            {
-                enumeratekeys(descriptor, System.out, cfStore.metadata);
-            }
-            else
-            {
-                if ((keys != null) && (keys.length > 0))
-                    export(descriptor, System.out, Arrays.asList(keys), excludes, cfStore.metadata);
-                else
-                    export(descriptor, excludes);
-            }
-        }
-        catch (IOException e)
-        {
-            // throwing exception outside main with broken pipe causes windows cmd to hang
-            e.printStackTrace(System.err);
-        }
-
-        System.exit(0);
-    }
-
-    private static void writeJSON(PrintStream out, Object value)
-    {
-        try
-        {
-            jsonMapper.writeValue(out, value);
-        }
-        catch (Exception e)
-        {
-            throw new RuntimeException(e.getMessage(), e);
-        }
-    }
-}


Mime
View raw message