cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xe...@apache.org
Subject [1/3] cassandra git commit: Add row offset support to SASI
Date Tue, 06 Sep 2016 05:19:17 GMT
Repository: cassandra
Updated Branches:
  refs/heads/trunk 3c95d4731 -> 7d857b46f


http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d857b46/test/unit/org/apache/cassandra/index/sasi/disk/TokenTreeTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/index/sasi/disk/TokenTreeTest.java b/test/unit/org/apache/cassandra/index/sasi/disk/TokenTreeTest.java
index 927e165..7c2498c 100644
--- a/test/unit/org/apache/cassandra/index/sasi/disk/TokenTreeTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/disk/TokenTreeTest.java
@@ -19,42 +19,31 @@ package org.apache.cassandra.index.sasi.disk;
 
 import java.io.File;
 import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.util.*;
 
 import com.google.common.collect.Iterators;
 import com.google.common.collect.PeekingIterator;
 
+import com.carrotsearch.hppc.cursors.LongObjectCursor;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.db.ClusteringComparator;
 import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.LongType;
 import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.index.sasi.disk.TokenTreeBuilder.EntryType;
-import org.apache.cassandra.index.sasi.utils.CombinedTerm;
-import org.apache.cassandra.index.sasi.utils.CombinedValue;
-import org.apache.cassandra.index.sasi.utils.MappedBuffer;
-import org.apache.cassandra.index.sasi.utils.RangeIterator;
-import org.apache.cassandra.db.marshal.LongType;
-import org.apache.cassandra.index.sasi.utils.RangeUnionIterator;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.io.util.SequentialWriterOption;
-import org.apache.cassandra.utils.MurmurHash;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.index.sasi.utils.*;
+import org.apache.cassandra.io.util.*;
 
 import junit.framework.Assert;
 
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
-import com.carrotsearch.hppc.LongOpenHashSet;
-import com.carrotsearch.hppc.LongSet;
-import com.carrotsearch.hppc.cursors.LongCursor;
-import com.google.common.base.Function;
 
 public class TokenTreeTest
 {
-    private static final Function<Long, DecoratedKey> KEY_CONVERTER = new KeyConverter();
+    private static final ClusteringComparator CLUSTERING_COMPARATOR = new ClusteringComparator(LongType.instance);
 
     @BeforeClass
     public static void setupDD()
@@ -62,14 +51,30 @@ public class TokenTreeTest
         DatabaseDescriptor.daemonInitialization();
     }
 
-    static LongSet singleOffset = new LongOpenHashSet() {{ add(1); }};
-    static LongSet bigSingleOffset = new LongOpenHashSet() {{ add(2147521562L); }};
-    static LongSet shortPackableCollision = new LongOpenHashSet() {{ add(2L); add(3L); }};
// can pack two shorts
-    static LongSet intPackableCollision = new LongOpenHashSet() {{ add(6L); add(((long) Short.MAX_VALUE)
+ 1); }}; // can pack int & short
-    static LongSet multiCollision =  new LongOpenHashSet() {{ add(3L); add(4L); add(5L);
}}; // can't pack
-    static LongSet unpackableCollision = new LongOpenHashSet() {{ add(((long) Short.MAX_VALUE)
+ 1); add(((long) Short.MAX_VALUE) + 2); }}; // can't pack
+    static KeyOffsets singleOffset = new KeyOffsets() {{ put(1L, KeyOffsets.asArray(10L));
}};
+    static KeyOffsets bigSingleOffset = new KeyOffsets() {{ put(2147521562L, KeyOffsets.asArray(10));
}};
+    static KeyOffsets shortPackableCollision = new KeyOffsets() {{
+        put(2L, KeyOffsets.asArray(10));
+        put(3L, KeyOffsets.asArray(10));
+    }}; // can pack two shorts
+    static KeyOffsets intPackableCollision = new KeyOffsets()
+    {{
+        put(6L, KeyOffsets.asArray(10));
+        put(((long) Short.MAX_VALUE) + 1, KeyOffsets.asArray(10));
+    }}; // can pack int & short
+    static KeyOffsets multiCollision = new KeyOffsets()
+    {{
+        put(3L, KeyOffsets.asArray(10));
+        put(4L, KeyOffsets.asArray(10));
+        put(5L, KeyOffsets.asArray(10));
+    }}; // can't pack
+    static KeyOffsets unpackableCollision = new KeyOffsets()
+    {{
+        put(((long) Short.MAX_VALUE) + 1, KeyOffsets.asArray(10));
+        put(((long) Short.MAX_VALUE) + 2, KeyOffsets.asArray(10));
+    }}; // can't pack
 
-    final static SortedMap<Long, LongSet> simpleTokenMap = new TreeMap<Long, LongSet>()
+    final static SortedMap<Long, KeyOffsets> simpleTokenMap = new TreeMap<Long,
KeyOffsets>()
     {{
             put(1L, bigSingleOffset); put(3L, shortPackableCollision); put(4L, intPackableCollision);
put(6L, singleOffset);
             put(9L, multiCollision); put(10L, unpackableCollision); put(12L, singleOffset);
put(13L, singleOffset);
@@ -81,18 +86,20 @@ public class TokenTreeTest
             put(121L, singleOffset); put(122L, singleOffset); put(123L, singleOffset); put(125L,
singleOffset);
     }};
 
-    final static SortedMap<Long, LongSet> bigTokensMap = new TreeMap<Long, LongSet>()
+    final static SortedMap<Long, KeyOffsets> bigTokensMap = new TreeMap<Long, KeyOffsets>()
     {{
             for (long i = 0; i < 1000000; i++)
                 put(i, singleOffset);
     }};
 
-    final static SortedMap<Long, LongSet> collidingTokensMap = new TreeMap<Long,
LongSet>()
+    final static SortedMap<Long, KeyOffsets> collidingTokensMap = new TreeMap<Long,
KeyOffsets>()
     {{
-            put(1L, singleOffset); put(7L, singleOffset); put(8L, singleOffset);
+        put(1L, singleOffset);
+        put(7L, singleOffset);
+        put(8L, singleOffset);
     }};
 
-    final static SortedMap<Long, LongSet> tokens = bigTokensMap;
+    final static SortedMap<Long, KeyOffsets> tokens = bigTokensMap;
 
     final static SequentialWriterOption DEFAULT_OPT = SequentialWriterOption.newBuilder().bufferSize(4096).build();
 
@@ -139,7 +146,7 @@ public class TokenTreeTest
     }
 
 
-    public void buildSerializeAndIterate(TokenTreeBuilder builder, SortedMap<Long, LongSet>
tokenMap) throws Exception
+    public void buildSerializeAndIterate(TokenTreeBuilder builder, SortedMap<Long, KeyOffsets>
tokenMap) throws Exception
     {
 
         builder.finish();
@@ -155,12 +162,12 @@ public class TokenTreeTest
         final RandomAccessReader reader = RandomAccessReader.open(treeFile);
         final TokenTree tokenTree = new TokenTree(new MappedBuffer(reader));
 
-        final Iterator<Token> tokenIterator = tokenTree.iterator(KEY_CONVERTER);
-        final Iterator<Map.Entry<Long, LongSet>> listIterator = tokenMap.entrySet().iterator();
+        final Iterator<Token> tokenIterator = tokenTree.iterator(KeyConverter.instance);
+        final Iterator<Map.Entry<Long, KeyOffsets>> listIterator = tokenMap.entrySet().iterator();
         while (tokenIterator.hasNext() && listIterator.hasNext())
         {
             Token treeNext = tokenIterator.next();
-            Map.Entry<Long, LongSet> listNext = listIterator.next();
+            Map.Entry<Long, KeyOffsets> listNext = listIterator.next();
 
             Assert.assertEquals(listNext.getKey(), treeNext.get());
             Assert.assertEquals(convert(listNext.getValue()), convert(treeNext));
@@ -193,15 +200,15 @@ public class TokenTreeTest
 
         for (long i = 0; i <= tokMax; i++)
         {
-            TokenTree.OnDiskToken result = tokenTree.get(i, KEY_CONVERTER);
+            TokenTree.OnDiskToken result = tokenTree.get(i, KeyConverter.instance);
             Assert.assertNotNull("failed to find object for token " + i, result);
 
-            LongSet found = result.getOffsets();
+            KeyOffsets found = result.getOffsets();
             Assert.assertEquals(1, found.size());
-            Assert.assertEquals(i, found.toArray()[0]);
+            Assert.assertEquals(i, found.iterator().next().key);
         }
 
-        Assert.assertNull("found missing object", tokenTree.get(tokMax + 10, KEY_CONVERTER));
+        Assert.assertNull("found missing object", tokenTree.get(tokMax + 10, KeyConverter.instance));
     }
 
     @Test
@@ -216,7 +223,7 @@ public class TokenTreeTest
         buildSerializeIterateAndSkip(new StaticTokenTreeBuilder(new FakeCombinedTerm(tokens)),
tokens);
     }
 
-    public void buildSerializeIterateAndSkip(TokenTreeBuilder builder, SortedMap<Long,
LongSet> tokens) throws Exception
+    public void buildSerializeIterateAndSkip(TokenTreeBuilder builder, SortedMap<Long,
KeyOffsets> tokens) throws Exception
     {
         builder.finish();
         final File treeFile = File.createTempFile("token-tree-iterate-test2", "tt");
@@ -231,7 +238,7 @@ public class TokenTreeTest
         final RandomAccessReader reader = RandomAccessReader.open(treeFile);
         final TokenTree tokenTree = new TokenTree(new MappedBuffer(reader));
 
-        final RangeIterator<Long, Token> treeIterator = tokenTree.iterator(KEY_CONVERTER);
+        final RangeIterator<Long, Token> treeIterator = tokenTree.iterator(KeyConverter.instance);
         final RangeIterator<Long, TokenWithOffsets> listIterator = new EntrySetSkippableIterator(tokens);
 
         long lastToken = 0L;
@@ -275,7 +282,7 @@ public class TokenTreeTest
         skipPastEnd(new StaticTokenTreeBuilder(new FakeCombinedTerm(simpleTokenMap)), simpleTokenMap);
     }
 
-    public void skipPastEnd(TokenTreeBuilder builder, SortedMap<Long, LongSet> tokens)
throws Exception
+    public void skipPastEnd(TokenTreeBuilder builder, SortedMap<Long, KeyOffsets> tokens)
throws Exception
     {
         builder.finish();
         final File treeFile = File.createTempFile("token-tree-skip-past-test", "tt");
@@ -288,7 +295,7 @@ public class TokenTreeTest
         }
 
         final RandomAccessReader reader = RandomAccessReader.open(treeFile);
-        final RangeIterator<Long, Token> tokenTree = new TokenTree(new MappedBuffer(reader)).iterator(KEY_CONVERTER);
+        final RangeIterator<Long, Token> tokenTree = new TokenTree(new MappedBuffer(reader)).iterator(KeyConverter.instance);
 
         tokenTree.skipTo(tokens.lastKey() + 10);
     }
@@ -313,8 +320,8 @@ public class TokenTreeTest
         TokenTree treeA = generateTree(min, max, isStatic);
         TokenTree treeB = generateTree(min, max, isStatic);
 
-        RangeIterator<Long, Token> a = treeA.iterator(new KeyConverter());
-        RangeIterator<Long, Token> b = treeB.iterator(new KeyConverter());
+        RangeIterator<Long, Token> a = treeA.iterator(KeyConverter.instance);
+        RangeIterator<Long, Token> b = treeB.iterator(KeyConverter.instance);
 
         long count = min;
         while (a.hasNext() && b.hasNext())
@@ -332,7 +339,8 @@ public class TokenTreeTest
             // should fail when trying to merge different tokens
             try
             {
-                tokenA.merge(new TokenWithOffsets(tokenA.get() + 1, convert(count)));
+                long l = tokenA.get();
+                tokenA.merge(new TokenWithOffsets(l + 1, convert(count)));
                 Assert.fail();
             }
             catch (IllegalArgumentException e)
@@ -341,8 +349,8 @@ public class TokenTreeTest
             }
 
             final Set<Long> offsets = new TreeSet<>();
-            for (DecoratedKey key : tokenA)
-                 offsets.add(LongType.instance.compose(key.getKey()));
+            for (RowKey key : tokenA)
+                offsets.add(LongType.instance.compose(key.decoratedKey.getKey()));
 
             Set<Long> expected = new TreeSet<>();
             {
@@ -373,7 +381,7 @@ public class TokenTreeTest
         testMergingOfEqualTokenTrees(bigTokensMap);
     }
 
-    public void testMergingOfEqualTokenTrees(SortedMap<Long, LongSet> tokensMap) throws
Exception
+    public void testMergingOfEqualTokenTrees(SortedMap<Long, KeyOffsets> tokensMap)
throws Exception
     {
         TokenTreeBuilder tokensA = new DynamicTokenTreeBuilder(tokensMap);
         TokenTreeBuilder tokensB = new DynamicTokenTreeBuilder(tokensMap);
@@ -386,8 +394,8 @@ public class TokenTreeTest
             public RangeIterator<Long, Token> getTokenIterator()
             {
                 RangeIterator.Builder<Long, Token> union = RangeUnionIterator.builder();
-                union.add(a.iterator(new KeyConverter()));
-                union.add(b.iterator(new KeyConverter()));
+                union.add(a.iterator(KeyConverter.instance));
+                union.add(b.iterator(KeyConverter.instance));
 
                 return union.build();
             }
@@ -395,31 +403,30 @@ public class TokenTreeTest
 
         TokenTree c = buildTree(tokensC);
         Assert.assertEquals(tokensMap.size(), c.getCount());
+        Iterator<Token> tokenIterator = c.iterator(KeyConverter.instance);
+        Iterator<Map.Entry<Long, KeyOffsets>> listIterator = tokensMap.entrySet().iterator();
 
-        Iterator<Token> tokenIterator = c.iterator(KEY_CONVERTER);
-        Iterator<Map.Entry<Long, LongSet>> listIterator = tokensMap.entrySet().iterator();
         while (tokenIterator.hasNext() && listIterator.hasNext())
         {
             Token treeNext = tokenIterator.next();
-            Map.Entry<Long, LongSet> listNext = listIterator.next();
+            Map.Entry<Long, KeyOffsets> listNext = listIterator.next();
 
             Assert.assertEquals(listNext.getKey(), treeNext.get());
             Assert.assertEquals(convert(listNext.getValue()), convert(treeNext));
         }
 
-        for (Map.Entry<Long, LongSet> entry : tokensMap.entrySet())
+        for (Map.Entry<Long, KeyOffsets> entry : tokensMap.entrySet())
         {
-            TokenTree.OnDiskToken result = c.get(entry.getKey(), KEY_CONVERTER);
+            TokenTree.OnDiskToken result = c.get(entry.getKey(), KeyConverter.instance);
             Assert.assertNotNull("failed to find object for token " + entry.getKey(), result);
-
-            LongSet found = result.getOffsets();
+            KeyOffsets found = result.getOffsets();
             Assert.assertEquals(entry.getValue(), found);
 
         }
     }
 
 
-    private static TokenTree buildTree(TokenTreeBuilder builder) throws Exception
+    private TokenTree buildTree(TokenTreeBuilder builder) throws Exception
     {
         builder.finish();
         final File treeFile = File.createTempFile("token-tree-", "db");
@@ -437,9 +444,9 @@ public class TokenTreeTest
 
     private static class EntrySetSkippableIterator extends RangeIterator<Long, TokenWithOffsets>
     {
-        private final PeekingIterator<Map.Entry<Long, LongSet>> elements;
+        private final PeekingIterator<Map.Entry<Long, KeyOffsets>> elements;
 
-        EntrySetSkippableIterator(SortedMap<Long, LongSet> elms)
+        EntrySetSkippableIterator(SortedMap<Long, KeyOffsets> elms)
         {
             super(elms.firstKey(), elms.lastKey(), elms.size());
             elements = Iterators.peekingIterator(elms.entrySet().iterator());
@@ -451,7 +458,7 @@ public class TokenTreeTest
             if (!elements.hasNext())
                 return endOfData();
 
-            Map.Entry<Long, LongSet> next = elements.next();
+            Map.Entry<Long, KeyOffsets> next = elements.next();
             return new TokenWithOffsets(next.getKey(), next.getValue());
         }
 
@@ -478,9 +485,9 @@ public class TokenTreeTest
 
     public static class FakeCombinedTerm extends CombinedTerm
     {
-        private final SortedMap<Long, LongSet> tokens;
+        private final SortedMap<Long, KeyOffsets> tokens;
 
-        public FakeCombinedTerm(SortedMap<Long, LongSet> tokens)
+        public FakeCombinedTerm(SortedMap<Long, KeyOffsets> tokens)
         {
             super(null, null);
             this.tokens = tokens;
@@ -494,9 +501,9 @@ public class TokenTreeTest
 
     public static class TokenMapIterator extends RangeIterator<Long, Token>
     {
-        public final Iterator<Map.Entry<Long, LongSet>> iterator;
+        public final Iterator<Map.Entry<Long, KeyOffsets>> iterator;
 
-        public TokenMapIterator(SortedMap<Long, LongSet> tokens)
+        public TokenMapIterator(SortedMap<Long, KeyOffsets> tokens)
         {
             super(tokens.firstKey(), tokens.lastKey(), tokens.size());
             iterator = tokens.entrySet().iterator();
@@ -507,7 +514,7 @@ public class TokenTreeTest
             if (!iterator.hasNext())
                 return endOfData();
 
-            Map.Entry<Long, LongSet> entry = iterator.next();
+            Map.Entry<Long, KeyOffsets> entry = iterator.next();
             return new TokenWithOffsets(entry.getKey(), entry.getValue());
         }
 
@@ -524,16 +531,16 @@ public class TokenTreeTest
 
     public static class TokenWithOffsets extends Token
     {
-        private final LongSet offsets;
+        private final KeyOffsets offsets;
 
-        public TokenWithOffsets(long token, final LongSet offsets)
+        public TokenWithOffsets(Long token, final KeyOffsets offsets)
         {
             super(token);
             this.offsets = offsets;
         }
 
         @Override
-        public LongSet getOffsets()
+        public KeyOffsets getOffsets()
         {
             return offsets;
         }
@@ -571,71 +578,56 @@ public class TokenTreeTest
         }
 
         @Override
-        public Iterator<DecoratedKey> iterator()
+        public Iterator<RowKey> iterator()
         {
-            List<DecoratedKey> keys = new ArrayList<>(offsets.size());
-            for (LongCursor offset : offsets)
-                 keys.add(dk(offset.value));
-
+            List<RowKey> keys = new ArrayList<>(offsets.size());
+            for (LongObjectCursor<long[]> offset : offsets)
+                for (long l : offset.value)
+                    keys.add(KeyConverter.instance.getRowKey(offset.key, l));
             return keys.iterator();
         }
     }
 
-    private static Set<DecoratedKey> convert(LongSet offsets)
+    private static Set<RowKey> convert(KeyOffsets offsets)
     {
-        Set<DecoratedKey> keys = new HashSet<>();
-        for (LongCursor offset : offsets)
-            keys.add(KEY_CONVERTER.apply(offset.value));
+        Set<RowKey> keys = new HashSet<>();
+        for (LongObjectCursor<long[]> offset : offsets)
+            for (long l : offset.value)
+                keys.add(new RowKey(KeyConverter.dk(offset.key),
+                                    KeyConverter.ck(l),
+                                    CLUSTERING_COMPARATOR));
 
         return keys;
     }
 
-    private static Set<DecoratedKey> convert(Token results)
+    private static Set<RowKey> convert(Token results)
     {
-        Set<DecoratedKey> keys = new HashSet<>();
-        for (DecoratedKey key : results)
+        Set<RowKey> keys = new HashSet<>();
+        for (RowKey key : results)
             keys.add(key);
 
         return keys;
     }
 
-    private static LongSet convert(long... values)
+    private static KeyOffsets convert(long... values)
     {
-        LongSet result = new LongOpenHashSet(values.length);
+        KeyOffsets result = new KeyOffsets(values.length);
         for (long v : values)
-            result.add(v);
+            result.put(v, KeyOffsets.asArray(v + 5));
 
         return result;
     }
 
-    private static class KeyConverter implements Function<Long, DecoratedKey>
-    {
-        @Override
-        public DecoratedKey apply(Long offset)
-        {
-            return dk(offset);
-        }
-    }
-
-    private static DecoratedKey dk(Long token)
-    {
-        ByteBuffer buf = ByteBuffer.allocate(8);
-        buf.putLong(token);
-        buf.flip();
-        Long hashed = MurmurHash.hash2_64(buf, buf.position(), buf.remaining(), 0);
-        return new BufferDecoratedKey(new Murmur3Partitioner.LongToken(hashed), buf);
-    }
-
-    private static TokenTree generateTree(final long minToken, final long maxToken, boolean
isStatic) throws IOException
+    private TokenTree generateTree(final long minToken, final long maxToken, boolean isStatic)
throws IOException
     {
-        final SortedMap<Long, LongSet> toks = new TreeMap<Long, LongSet>()
+        final SortedMap<Long, KeyOffsets> toks = new TreeMap<Long, KeyOffsets>()
         {{
-                for (long i = minToken; i <= maxToken; i++)
-                {
-                    LongSet offsetSet = new LongOpenHashSet();
-                    offsetSet.add(i);
-                    put(i, offsetSet);
-                }
+            for (long i = minToken; i <= maxToken; i++)
+            {
+                KeyOffsets offsetSet = new KeyOffsets();
+                offsetSet.put(i, KeyOffsets.asArray(i + 5));
+                put(i, offsetSet);
+            }
         }};
 
         final TokenTreeBuilder builder = isStatic ? new StaticTokenTreeBuilder(new FakeCombinedTerm(toks))
: new DynamicTokenTreeBuilder(toks);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d857b46/test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java b/test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java
index e388cd4..f89dd6c 100644
--- a/test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/plan/OperationTest.java
@@ -47,7 +47,7 @@ import org.junit.*;
 
 public class OperationTest extends SchemaLoader
 {
-    private static final String KS_NAME = "sasi";
+    private static final String KS_NAME = "operation_test";
     private static final String CF_NAME = "test_cf";
     private static final String CLUSTERING_CF_NAME = "clustering_test_cf";
     private static final String STATIC_CF_NAME = "static_sasi_test_cf";

http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d857b46/test/unit/org/apache/cassandra/index/sasi/utils/KeyConverter.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/index/sasi/utils/KeyConverter.java b/test/unit/org/apache/cassandra/index/sasi/utils/KeyConverter.java
new file mode 100644
index 0000000..7de502a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sasi/utils/KeyConverter.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sasi.utils;
+
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.dht.*;
+import org.apache.cassandra.index.sasi.*;
+import org.apache.cassandra.index.sasi.disk.*;
+import org.apache.cassandra.utils.*;
+
+public class KeyConverter implements KeyFetcher
+{
+    public final static KeyConverter instance = new KeyConverter();
+
+    KeyConverter()
+    {}
+
+    @Override
+    public DecoratedKey getPartitionKey(long offset)
+    {
+        return dk(offset);
+    }
+
+    @Override
+    public Clustering getClustering(long offset)
+    {
+        return ck(offset);
+    }
+
+    @Override
+
+    public RowKey getRowKey(long partitionOffset, long rowOffset)
+    {
+        return new RowKey(getPartitionKey(partitionOffset), getClustering(rowOffset), new
ClusteringComparator(LongType.instance));
+    }
+
+    public static DecoratedKey dk(long partitionOffset)
+    {
+        ByteBuffer buf = ByteBuffer.allocate(8);
+        buf.putLong(partitionOffset);
+        buf.flip();
+        Long hashed = MurmurHash.hash2_64(buf, buf.position(), buf.remaining(), 0);
+        return new BufferDecoratedKey(new Murmur3Partitioner.LongToken(hashed), buf);
+    }
+
+    public static Clustering ck(long offset)
+    {
+        return Clustering.make(ByteBufferUtil.bytes(offset));
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d857b46/test/unit/org/apache/cassandra/index/sasi/utils/LongIterator.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/index/sasi/utils/LongIterator.java b/test/unit/org/apache/cassandra/index/sasi/utils/LongIterator.java
index 205d28f..4f80a1c 100644
--- a/test/unit/org/apache/cassandra/index/sasi/utils/LongIterator.java
+++ b/test/unit/org/apache/cassandra/index/sasi/utils/LongIterator.java
@@ -26,6 +26,8 @@ import java.util.List;
 import com.carrotsearch.hppc.LongOpenHashSet;
 import com.carrotsearch.hppc.LongSet;
 import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.index.sasi.disk.KeyOffsets;
+import org.apache.cassandra.index.sasi.disk.RowKey;
 import org.apache.cassandra.index.sasi.disk.Token;
 
 public class LongIterator extends RangeIterator<Long, Token>
@@ -82,13 +84,13 @@ public class LongIterator extends RangeIterator<Long, Token>
         }
 
         @Override
-        public LongSet getOffsets()
+        public KeyOffsets getOffsets()
         {
-            return new LongOpenHashSet(4);
+            return new KeyOffsets(4);
         }
 
         @Override
-        public Iterator<DecoratedKey> iterator()
+        public Iterator<RowKey> iterator()
         {
             return Collections.emptyIterator();
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d857b46/test/unit/org/apache/cassandra/index/sasi/utils/RangeUnionIteratorTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/index/sasi/utils/RangeUnionIteratorTest.java b/test/unit/org/apache/cassandra/index/sasi/utils/RangeUnionIteratorTest.java
index f69086b..4819c0d 100644
--- a/test/unit/org/apache/cassandra/index/sasi/utils/RangeUnionIteratorTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/utils/RangeUnionIteratorTest.java
@@ -31,6 +31,23 @@ import static org.apache.cassandra.index.sasi.utils.LongIterator.convert;
 public class RangeUnionIteratorTest
 {
     @Test
+    public void mergingOfEqualTokensTest()
+    {
+        RangeUnionIterator.Builder<Long, Token> builder = RangeUnionIterator.builder();
+
+        int size = 1000000;
+        final long[] arr = new long[size];
+        for (int i = 0; i < size; i++)
+            arr[i] = i;
+
+        builder.add(new LongIterator(arr));
+        builder.add(new LongIterator(arr));
+
+        Assert.assertEquals(convert(arr), convert(builder.build()));
+    }
+
+
+    @Test
     public void testNoOverlappingValues()
     {
         RangeUnionIterator.Builder<Long, Token> builder = RangeUnionIterator.builder();


Mime
View raw message