cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sn...@apache.org
Subject [2/3] cassandra git commit: Remove IndexInfo cache from FileIndexInfoRetriever
Date Tue, 04 Oct 2016 13:15:55 GMT
Remove IndexInfo cache from FileIndexInfoRetriever

patch by Yasuharu Goto; reviewed by Robert Stupp for CASSANDRA-12731


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/033f2565
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/033f2565
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/033f2565

Branch: refs/heads/trunk
Commit: 033f2565ef25bc764141dc0fc64088fab6de89c9
Parents: 433dd1c
Author: Yasuharu Goto <matope.ono@gmail.com>
Authored: Tue Oct 4 14:40:43 2016 +0200
Committer: Robert Stupp <snazy@snazy-ds15.fritz.box>
Committed: Tue Oct 4 14:40:43 2016 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../org/apache/cassandra/db/RowIndexEntry.java  | 35 ++++----------------
 .../io/sstable/LargePartitionsTest.java         | 22 ++++++++++--
 3 files changed, 27 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/033f2565/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b503fd9..64743b4 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 3.10
+ * Remove IndexInfo cache from FileIndexInfoRetriever (CASSANDRA-12731)
  * ColumnIndex does not reuse buffer (CASSANDRA-12502)
  * cdc column addition still breaks schema migration tasks (CASSANDRA-12697)
  * Upgrade metrics-reporter dependencies (CASSANDRA-12089)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/033f2565/src/java/org/apache/cassandra/db/RowIndexEntry.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/RowIndexEntry.java b/src/java/org/apache/cassandra/db/RowIndexEntry.java
index 317b2bd..e620dc0 100644
--- a/src/java/org/apache/cassandra/db/RowIndexEntry.java
+++ b/src/java/org/apache/cassandra/db/RowIndexEntry.java
@@ -19,7 +19,6 @@ package org.apache.cassandra.db;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.Arrays;
 import java.util.List;
 
 import com.codahale.metrics.Histogram;
@@ -556,7 +555,7 @@ public class RowIndexEntry<T> implements IMeasurableMemory
 
         private LegacyIndexInfoRetriever(long indexFilePosition, int[] offsets, FileDataInput
reader, IndexInfo.Serializer idxInfoSerializer)
         {
-            super(indexFilePosition, offsets.length, reader, idxInfoSerializer);
+            super(indexFilePosition, reader, idxInfoSerializer);
             this.offsets = offsets;
         }
 
@@ -889,7 +888,7 @@ public class RowIndexEntry<T> implements IMeasurableMemory
                                             VIntCoding.computeUnsignedVIntSize(indexedPartSize
+ fieldsSerializedSize) +
                                             fieldsSerializedSize,
                                             offsetsOffset - fieldsSerializedSize,
-                                            columnsIndexCount, indexFile.createReader(),
idxInfoSerializer);
+                                            indexFile.createReader(), idxInfoSerializer);
         }
 
         @Override
@@ -954,17 +953,15 @@ public class RowIndexEntry<T> implements IMeasurableMemory
     {
         private final int offsetsOffset;
 
-        private ShallowInfoRetriever(long indexInfoFilePosition, int offsetsOffset, int indexCount,
+        private ShallowInfoRetriever(long indexInfoFilePosition, int offsetsOffset,
                                      FileDataInput indexReader, ISerializer<IndexInfo>
idxInfoSerializer)
         {
-            super(indexInfoFilePosition, indexCount, indexReader, idxInfoSerializer);
+            super(indexInfoFilePosition, indexReader, idxInfoSerializer);
             this.offsetsOffset = offsetsOffset;
         }
 
         IndexInfo fetchIndex(int index) throws IOException
         {
-            assert index >= 0 && index < indexCount;
-
             retrievals++;
 
             // seek to position in "offsets to IndexInfo" table
@@ -998,46 +995,26 @@ public class RowIndexEntry<T> implements IMeasurableMemory
     private abstract static class FileIndexInfoRetriever implements IndexInfoRetriever
     {
         final long indexInfoFilePosition;
-        final int indexCount;
         final ISerializer<IndexInfo> idxInfoSerializer;
         final FileDataInput indexReader;
         int retrievals;
 
-        private IndexInfo[] lastIndexes;
-
         /**
          *
          * @param indexInfoFilePosition offset of first serialized {@link IndexInfo} object
-         * @param indexCount number of {@link IndexInfo} objects
          * @param indexReader file data input to access the index file, closed by this instance
          * @param idxInfoSerializer the index serializer to deserialize {@link IndexInfo}
objects
          */
-        FileIndexInfoRetriever(long indexInfoFilePosition, int indexCount, FileDataInput
indexReader, ISerializer<IndexInfo> idxInfoSerializer)
+        FileIndexInfoRetriever(long indexInfoFilePosition, FileDataInput indexReader, ISerializer<IndexInfo>
idxInfoSerializer)
         {
             this.indexInfoFilePosition = indexInfoFilePosition;
-            this.indexCount = indexCount;
             this.idxInfoSerializer = idxInfoSerializer;
             this.indexReader = indexReader;
         }
 
         public final IndexInfo columnsIndex(int index) throws IOException
         {
-            if (lastIndexes != null
-                && lastIndexes.length > index && lastIndexes[index] !=
null)
-            {
-                // return a previously read/deserialized IndexInfo
-                return lastIndexes[index];
-            }
-
-            if (lastIndexes == null)
-                lastIndexes = new IndexInfo[index + 1];
-            else if (lastIndexes.length <= index)
-                lastIndexes = Arrays.copyOf(lastIndexes, index + 1);
-
-            IndexInfo indexInfo = fetchIndex(index);
-            lastIndexes[index] = indexInfo;
-
-            return indexInfo;
+            return fetchIndex(index);
         }
 
         abstract IndexInfo fetchIndex(int index) throws IOException;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/033f2565/test/unit/org/apache/cassandra/io/sstable/LargePartitionsTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/LargePartitionsTest.java b/test/unit/org/apache/cassandra/io/sstable/LargePartitionsTest.java
index 5536efd..45fd712 100644
--- a/test/unit/org/apache/cassandra/io/sstable/LargePartitionsTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/LargePartitionsTest.java
@@ -15,9 +15,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.cassandra.io.sstable;
 
+import java.util.Iterator;
 import java.util.concurrent.ThreadLocalRandom;
 
 import org.junit.Ignore;
@@ -26,6 +26,7 @@ import org.junit.runner.RunWith;
 
 import org.apache.cassandra.OrderedJUnit4ClassRunner;
 import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.metrics.CacheMetrics;
 import org.apache.cassandra.service.CacheService;
 
@@ -49,7 +50,7 @@ public class LargePartitionsTest extends CQLTester
         long t0 = System.currentTimeMillis();
         measured.measure();
         long t = System.currentTimeMillis() - t0;
-        System.out.println(name + " took " + t + " ms");
+        System.out.println("LargePartitionsTest-measured: " + name + " took " + t + " ms");
     }
 
     private static String randomText(int bytes)
@@ -96,6 +97,9 @@ public class LargePartitionsTest extends CQLTester
         measured("SELECTs 1 for " + name, () -> selects(partitionKBytes, totalKBytes));
 
         measured("SELECTs 2 for " + name, () -> selects(partitionKBytes, totalKBytes));
+
+        CacheService.instance.keyCache.clear();
+        measured("Scan for " + name, () -> scan(partitionKBytes, totalKBytes));
     }
 
     private void selects(long partitionKBytes, long totalKBytes) throws Throwable
@@ -113,6 +117,20 @@ public class LargePartitionsTest extends CQLTester
         keyCacheMetrics("after all selects");
     }
 
+    private void scan(long partitionKBytes, long totalKBytes) throws Throwable
+    {
+        long pk = ThreadLocalRandom.current().nextLong(totalKBytes / partitionKBytes) * partitionKBytes;
+        Iterator<UntypedResultSet.Row> iter = execute("SELECT val FROM %s WHERE pk=?",
Long.toBinaryString(pk)).iterator();
+        int i = 0;
+        while (iter.hasNext())
+        {
+            iter.next();
+            if (i++ % 1000 == 0)
+                keyCacheMetrics("after " + i + " iteration");
+        }
+        keyCacheMetrics("after all iteration");
+    }
+
     private static void keyCacheMetrics(String title)
     {
         CacheMetrics metrics = CacheService.instance.keyCache.getMetrics();


Mime
View raw message