cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From adelap...@apache.org
Subject [5/8] cassandra git commit: Merge branch 'cassandra-3.0' into cassandra-3.11
Date Tue, 08 Aug 2017 14:27:41 GMT
Merge branch 'cassandra-3.0' into cassandra-3.11


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/47a2839b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/47a2839b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/47a2839b

Branch: refs/heads/trunk
Commit: 47a2839bf094a7fa9bad6de140fba486756c49bf
Parents: 15abe2d 3960260
Author: Andrés de la Peña <a.penya.garcia@gmail.com>
Authored: Tue Aug 8 15:01:04 2017 +0100
Committer: Andrés de la Peña <a.penya.garcia@gmail.com>
Committed: Tue Aug 8 15:01:04 2017 +0100

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../cassandra/config/ColumnDefinition.java      | 19 ++++++++---
 .../apache/cassandra/db/rows/AbstractCell.java  |  2 +-
 .../org/apache/cassandra/db/rows/BTreeRow.java  |  2 +-
 .../apache/cassandra/tools/JsonTransformer.java | 31 ++++++++++++++++--
 .../org/apache/cassandra/cql3/ViewTest.java     | 33 ++++++++++++++++++++
 .../cassandra/index/sasi/SASIIndexTest.java     | 14 +++++++++
 7 files changed, 94 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/47a2839b/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index b8c4bde,1525289..b778df6
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,9 -1,5 +1,10 @@@
 -3.0.15
 +3.11.1
 + * "ignore" option is ignored in sstableloader (CASSANDRA-13721)
 + * Deadlock in AbstractCommitLogSegmentManager (CASSANDRA-13652)
 + * Duplicate the buffer before passing it to analyser in SASI operation (CASSANDRA-13512)
 + * Properly evict pstmts from prepared statements cache (CASSANDRA-13641)
 +Merged from 3.0:
+  * Fix ColumnDefinition.cellValueType() for non-frozen collection and change SSTabledump
to use type.toJSONString() (CASSANDRA-13573)
   * Skip materialized view addition if the base table doesn't exist (CASSANDRA-13737)
   * Drop table should remove corresponding entries in dropped_columns table (CASSANDRA-13730)
   * Log warn message until legacy auth tables have been migrated (CASSANDRA-13371)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/47a2839b/src/java/org/apache/cassandra/config/ColumnDefinition.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/config/ColumnDefinition.java
index ea508d2,6a0f530..159ea0c
--- a/src/java/org/apache/cassandra/config/ColumnDefinition.java
+++ b/src/java/org/apache/cassandra/config/ColumnDefinition.java
@@@ -446,197 -398,17 +448,206 @@@ public class ColumnDefinition extends C
       */
      public AbstractType<?> cellValueType()
      {
-         return type instanceof CollectionType
-              ? ((CollectionType)type).valueComparator()
-              : type;
+         assert !(type instanceof UserType && type.isMultiCell());
+         return type instanceof CollectionType && type.isMultiCell()
+                 ? ((CollectionType)type).valueComparator()
+                 : type;
+     }
+ 
+ 
+     public boolean isCounterColumn()
+     {
+         if (type instanceof CollectionType) // for thrift
+             return ((CollectionType) type).valueComparator().isCounter();
+         return type.isCounter();
      }
 +
 +    public Selector.Factory newSelectorFactory(CFMetaData cfm, AbstractType<?> expectedType,
List<ColumnDefinition> defs, VariableSpecifications boundNames) throws InvalidRequestException
 +    {
 +        return SimpleSelector.newFactory(this, addAndGetIndex(this, defs));
 +    }
 +
 +    public AbstractType<?> getExactTypeIfKnown(String keyspace)
 +    {
 +        return type;
 +    }
 +
 +    /**
 +     * Because Thrift-created tables may have a non-text comparator, we cannot determine
the proper 'key' until
 +     * we know the comparator. ColumnDefinition.Raw is a placeholder that can be converted
to a real ColumnIdentifier
 +     * once the comparator is known with prepare(). This should only be used with identifiers
that are actual
 +     * column names. See CASSANDRA-8178 for more background.
 +     */
 +    public static abstract class Raw extends Selectable.Raw
 +    {
 +        /**
 +         * Creates a {@code ColumnDefinition.Raw} from an unquoted identifier string.
 +         */
 +        public static Raw forUnquoted(String text)
 +        {
 +            return new Literal(text, false);
 +        }
 +
 +        /**
 +         * Creates a {@code ColumnDefinition.Raw} from a quoted identifier string.
 +         */
 +        public static Raw forQuoted(String text)
 +        {
 +            return new Literal(text, true);
 +        }
 +
 +        /**
 +         * Creates a {@code ColumnDefinition.Raw} from a pre-existing {@code ColumnDefinition}
 +         * (useful in the rare cases where we already have the column but need
 +         * a {@code ColumnDefinition.Raw} for typing purposes).
 +         */
 +        public static Raw forColumn(ColumnDefinition column)
 +        {
 +            return new ForColumn(column);
 +        }
 +
 +        /**
 +         * Get the identifier corresponding to this raw column, without assuming this is
an
 +         * existing column (unlike {@link #prepare}).
 +         */
 +        public abstract ColumnIdentifier getIdentifier(CFMetaData cfm);
 +
 +        public abstract String rawText();
 +
 +        @Override
 +        public abstract ColumnDefinition prepare(CFMetaData cfm);
 +
 +        @Override
 +        public boolean processesSelection()
 +        {
 +            return false;
 +        }
 +
 +        @Override
 +        public final int hashCode()
 +        {
 +            return toString().hashCode();
 +        }
 +
 +        @Override
 +        public final boolean equals(Object o)
 +        {
 +            if(!(o instanceof Raw))
 +                return false;
 +
 +            Raw that = (Raw)o;
 +            return this.toString().equals(that.toString());
 +        }
 +
 +        private static class Literal extends Raw
 +        {
 +            private final String text;
 +
 +            public Literal(String rawText, boolean keepCase)
 +            {
 +                this.text =  keepCase ? rawText : rawText.toLowerCase(Locale.US);
 +            }
 +
 +            public ColumnIdentifier getIdentifier(CFMetaData cfm)
 +            {
 +                if (!cfm.isStaticCompactTable())
 +                    return ColumnIdentifier.getInterned(text, true);
 +
 +                AbstractType<?> thriftColumnNameType = cfm.thriftColumnNameType();
 +                if (thriftColumnNameType instanceof UTF8Type)
 +                    return ColumnIdentifier.getInterned(text, true);
 +
 +                // We have a Thrift-created table with a non-text comparator. Check if we
have a match column, otherwise assume we should use
 +                // thriftColumnNameType
 +                ByteBuffer bufferName = ByteBufferUtil.bytes(text);
 +                for (ColumnDefinition def : cfm.allColumns())
 +                {
 +                    if (def.name.bytes.equals(bufferName))
 +                        return def.name;
 +                }
 +                return ColumnIdentifier.getInterned(thriftColumnNameType, thriftColumnNameType.fromString(text),
text);
 +            }
 +
 +            public ColumnDefinition prepare(CFMetaData cfm)
 +            {
 +                if (!cfm.isStaticCompactTable())
 +                    return find(cfm);
 +
 +                AbstractType<?> thriftColumnNameType = cfm.thriftColumnNameType();
 +                if (thriftColumnNameType instanceof UTF8Type)
 +                    return find(cfm);
 +
 +                // We have a Thrift-created table with a non-text comparator. Check if we
have a match column, otherwise assume we should use
 +                // thriftColumnNameType
 +                ByteBuffer bufferName = ByteBufferUtil.bytes(text);
 +                for (ColumnDefinition def : cfm.allColumns())
 +                {
 +                    if (def.name.bytes.equals(bufferName))
 +                        return def;
 +                }
 +                return find(thriftColumnNameType.fromString(text), cfm);
 +            }
 +
 +            private ColumnDefinition find(CFMetaData cfm)
 +            {
 +                return find(ByteBufferUtil.bytes(text), cfm);
 +            }
 +
 +            private ColumnDefinition find(ByteBuffer id, CFMetaData cfm)
 +            {
 +                ColumnDefinition def = cfm.getColumnDefinition(id);
 +                if (def == null)
 +                    throw new InvalidRequestException(String.format("Undefined column name
%s", toString()));
 +                return def;
 +            }
 +
 +            public String rawText()
 +            {
 +                return text;
 +            }
 +
 +            @Override
 +            public String toString()
 +            {
 +                return ColumnIdentifier.maybeQuote(text);
 +            }
 +        }
 +
 +        // Use internally in the rare case where we need a ColumnDefinition.Raw for type-checking
but
 +        // actually already have the column itself.
 +        private static class ForColumn extends Raw
 +        {
 +            private final ColumnDefinition column;
 +
 +            private ForColumn(ColumnDefinition column)
 +            {
 +                this.column = column;
 +            }
 +
 +            public ColumnIdentifier getIdentifier(CFMetaData cfm)
 +            {
 +                return column.name;
 +            }
 +
 +            public ColumnDefinition prepare(CFMetaData cfm)
 +            {
 +                assert cfm.getColumnDefinition(column.name) != null; // Sanity check that
we're not doing something crazy
 +                return column;
 +            }
 +
 +            public String rawText()
 +            {
 +                return column.name.toString();
 +            }
 +
 +            @Override
 +            public String toString()
 +            {
 +                return column.name.toCQLString();
 +            }
 +        }
 +    }
 +
 +
 +
  }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/47a2839b/src/java/org/apache/cassandra/db/rows/AbstractCell.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/rows/AbstractCell.java
index 54bd9e8,7e93c2e..54c8f24
--- a/src/java/org/apache/cassandra/db/rows/AbstractCell.java
+++ b/src/java/org/apache/cassandra/db/rows/AbstractCell.java
@@@ -44,81 -40,6 +44,81 @@@ public abstract class AbstractCell exte
          super(column);
      }
  
 +    public boolean isCounterCell()
 +    {
-         return !isTombstone() && column.cellValueType().isCounter();
++        return !isTombstone() && column.isCounterColumn();
 +    }
 +
 +    public boolean isLive(int nowInSec)
 +    {
 +        return localDeletionTime() == NO_DELETION_TIME || (ttl() != NO_TTL && nowInSec
< localDeletionTime());
 +    }
 +
 +    public boolean isTombstone()
 +    {
 +        return localDeletionTime() != NO_DELETION_TIME && ttl() == NO_TTL;
 +    }
 +
 +    public boolean isExpiring()
 +    {
 +        return ttl() != NO_TTL;
 +    }
 +
 +    public Cell markCounterLocalToBeCleared()
 +    {
 +        if (!isCounterCell())
 +            return this;
 +
 +        ByteBuffer value = value();
 +        ByteBuffer marked = CounterContext.instance().markLocalToBeCleared(value);
 +        return marked == value ? this : new BufferCell(column, timestamp(), ttl(), localDeletionTime(),
marked, path());
 +    }
 +
 +    public Cell purge(DeletionPurger purger, int nowInSec)
 +    {
 +        if (!isLive(nowInSec))
 +        {
 +            if (purger.shouldPurge(timestamp(), localDeletionTime()))
 +                return null;
 +
 +            // We slightly hijack purging to convert expired but not purgeable columns to
tombstones. The reason we do that is
 +            // that once a column has expired it is equivalent to a tombstone but actually
using a tombstone is more compact since
 +            // we don't keep the column value. The reason we do it here is that 1) it's
somewhat related to dealing with tombstones
 +            // so hopefully not too surprising and 2) we want to this and purging at the
same places, so it's simpler/more efficient
 +            // to do both here.
 +            if (isExpiring())
 +            {
 +                // Note that as long as the expiring column and the tombstone put together
live longer than GC grace seconds,
 +                // we'll fulfil our responsibility to repair. See discussion at
 +                // http://cassandra-user-incubator-apache-org.3065146.n2.nabble.com/repair-compaction-and-tombstone-rows-td7583481.html
 +                return BufferCell.tombstone(column, timestamp(), localDeletionTime() - ttl(),
path()).purge(purger, nowInSec);
 +            }
 +        }
 +        return this;
 +    }
 +
 +    public Cell copy(AbstractAllocator allocator)
 +    {
 +        CellPath path = path();
 +        return new BufferCell(column, timestamp(), ttl(), localDeletionTime(), allocator.clone(value()),
path == null ? null : path.copy(allocator));
 +    }
 +
 +    // note: while the cell returned may be different, the value is the same, so if the
value is offheap it must be referenced inside a guarded context (or copied)
 +    public Cell updateAllTimestamp(long newTimestamp)
 +    {
 +        return new BufferCell(column, isTombstone() ? newTimestamp - 1 : newTimestamp, ttl(),
localDeletionTime(), value(), path());
 +    }
 +
 +    public int dataSize()
 +    {
 +        CellPath path = path();
 +        return TypeSizes.sizeof(timestamp())
 +               + TypeSizes.sizeof(ttl())
 +               + TypeSizes.sizeof(localDeletionTime())
 +               + value().remaining()
 +               + (path == null ? 0 : path.dataSize());
 +    }
 +
      public void digest(MessageDigest digest)
      {
          digest.update(value().duplicate());

http://git-wip-us.apache.org/repos/asf/cassandra/blob/47a2839b/src/java/org/apache/cassandra/db/rows/BTreeRow.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/47a2839b/src/java/org/apache/cassandra/tools/JsonTransformer.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/tools/JsonTransformer.java
index 360e8ff,5c32035..c3a0a17
--- a/src/java/org/apache/cassandra/tools/JsonTransformer.java
+++ b/src/java/org/apache/cassandra/tools/JsonTransformer.java
@@@ -46,6 -50,7 +47,7 @@@ import org.apache.cassandra.db.rows.Row
  import org.apache.cassandra.db.rows.Unfiltered;
  import org.apache.cassandra.db.rows.UnfilteredRowIterator;
  import org.apache.cassandra.io.sstable.ISSTableScanner;
 -import org.apache.cassandra.transport.Server;
++import org.apache.cassandra.transport.ProtocolVersion;
  import org.apache.cassandra.utils.ByteBufferUtil;
  import org.codehaus.jackson.JsonFactory;
  import org.codehaus.jackson.JsonGenerator;
@@@ -405,9 -411,9 +407,10 @@@ public final class JsonTransforme
              objectIndenter.setCompact(true);
              json.writeFieldName("name");
              AbstractType<?> type = cell.column().type;
++            AbstractType<?> cellType = null;
              json.writeString(cell.column().name.toCQLString());
  
-             if (cell.path() != null && cell.path().size() > 0)
+             if (type.isCollection() && type.isMultiCell()) // non-frozen collection
              {
                  CollectionType ct = (CollectionType) type;
                  json.writeFieldName("path");
@@@ -419,6 -425,6 +422,30 @@@
                  }
                  json.writeEndArray();
                  arrayIndenter.setCompact(false);
++
++                cellType = cell.column().cellValueType();
++            }
++            else if (type.isUDT() && type.isMultiCell()) // non-frozen udt
++            {
++                UserType ut = (UserType) type;
++                json.writeFieldName("path");
++                arrayIndenter.setCompact(true);
++                json.writeStartArray();
++                for (int i = 0; i < cell.path().size(); i++)
++                {
++                    Short fieldPosition = ut.nameComparator().compose(cell.path().get(i));
++                    json.writeString(ut.fieldNameAsString(fieldPosition));
++                }
++                json.writeEndArray();
++                arrayIndenter.setCompact(false);
++
++                // cellType of udt
++                Short fieldPosition = ((UserType) type).nameComparator().compose(cell.path().get(0));
++                cellType = ((UserType) type).fieldType(fieldPosition);
++            }
++            else
++            {
++                cellType = cell.column().cellValueType();
              }
              if (cell.isTombstone())
              {
@@@ -433,7 -439,7 +460,7 @@@
              else
              {
                  json.writeFieldName("value");
-                 json.writeString(cell.column().cellValueType().getString(cell.value()));
 -                json.writeRawValue(cell.column().cellValueType().toJSONString(cell.value(),
Server.CURRENT_VERSION));
++                json.writeRawValue(cellType.toJSONString(cell.value(), ProtocolVersion.CURRENT));
              }
              if (liveInfo.isEmpty() || cell.timestamp() != liveInfo.timestamp())
              {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/47a2839b/test/unit/org/apache/cassandra/cql3/ViewTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/ViewTest.java
index 6f6e04d,f8f8c9f..0853562
--- a/test/unit/org/apache/cassandra/cql3/ViewTest.java
+++ b/test/unit/org/apache/cassandra/cql3/ViewTest.java
@@@ -777,6 -788,6 +777,39 @@@ public class ViewTest extends CQLTeste
      }
  
      @Test
++    public void testFrozenCollectionsWithComplicatedInnerType() throws Throwable
++    {
++        createTable("CREATE TABLE %s (k int, intval int,  listval frozen<list<tuple<text,text>>>,
PRIMARY KEY (k))");
++
++        execute("USE " + keyspace());
++        executeNet(protocolVersion, "USE " + keyspace());
++
++        createView("mv",
++                   "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE k IS NOT NULL
AND listval IS NOT NULL PRIMARY KEY (k, listval)");
++
++        updateView("INSERT INTO %s (k, intval, listval) VALUES (?, ?, fromJson(?))",
++                   0,
++                   0,
++                   "[[\"a\",\"1\"], [\"b\",\"2\"], [\"c\",\"3\"]]");
++
++        // verify input
++        assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0),
++                   row(0, list(tuple("a", "1"), tuple("b", "2"), tuple("c", "3"))));
++        assertRows(execute("SELECT k, listval from mv"),
++                   row(0, list(tuple("a", "1"), tuple("b", "2"), tuple("c", "3"))));
++
++        // update listval with the same value and it will be compared in view generator
++        updateView("INSERT INTO %s (k, listval) VALUES (?, fromJson(?))",
++                   0,
++                   "[[\"a\",\"1\"], [\"b\",\"2\"], [\"c\",\"3\"]]");
++        // verify result
++        assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0),
++                   row(0, list(tuple("a", "1"), tuple("b", "2"), tuple("c", "3"))));
++        assertRows(execute("SELECT k, listval from mv"),
++                   row(0, list(tuple("a", "1"), tuple("b", "2"), tuple("c", "3"))));
++    }
++
++    @Test
      public void testUpdate() throws Throwable
      {
          createTable("CREATE TABLE %s (" +


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


Mime
View raw message