cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vi...@apache.org
Subject [1/4] git commit: additional check for offheap bloom filter size patch by Vijay; reviewed by jbellis for CASSANDRA-5903
Date Thu, 22 Aug 2013 02:25:07 GMT
Updated Branches:
  refs/heads/trunk 5e33b9b03 -> 74492ef27


additional check for offheap bloom filter size
patch by Vijay; reviewed by jbellis for CASSANDRA-5903


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9bb4d93e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9bb4d93e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9bb4d93e

Branch: refs/heads/trunk
Commit: 9bb4d93e3c8471012a39cbb0a1be5b082115a42b
Parents: 4bc8c89
Author: Vijay Parthasarathy <vijay2win@gmail.com>
Authored: Wed Aug 21 19:21:28 2013 -0700
Committer: Vijay Parthasarathy <vijay2win@gmail.com>
Committed: Wed Aug 21 19:21:28 2013 -0700

----------------------------------------------------------------------
 .../cassandra/utils/obs/OffHeapBitSet.java      |  5 ++-
 .../apache/cassandra/utils/BloomFilterTest.java | 36 ++++++++++++++++++++
 2 files changed, 40 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/9bb4d93e/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java b/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java
index fb32043..c103835 100644
--- a/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java
+++ b/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java
@@ -36,9 +36,12 @@ public class OffHeapBitSet implements IBitSet
     public OffHeapBitSet(long numBits)
     {
         // OpenBitSet.bits2words calculation is there for backward compatibility.
-        long byteCount = OpenBitSet.bits2words(numBits) * 8L;
+        long wordCount = OpenBitSet.bits2words(numBits);
+        if (wordCount > Integer.MAX_VALUE)
+            throw new UnsupportedOperationException("Bloom filter size is > 16GB, reduce
the bloom_filter_fp_chance");
         try
         {
+            long byteCount = wordCount * 8L;
             bytes = RefCountedMemory.allocate(byteCount);
         }
         catch (OutOfMemoryError e)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9bb4d93e/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
index bb0865f..562e3ea 100644
--- a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
+++ b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java
@@ -20,16 +20,25 @@ package org.apache.cassandra.utils;
 
 import java.io.ByteArrayInputStream;
 import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Set;
 
+import junit.framework.Assert;
+
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.FilterFactory.Type;
 
 public class BloomFilterTest
 {
@@ -138,4 +147,31 @@ public class BloomFilterTest
     {
         testManyHashes(FilterTestHelper.randomKeys());
     }
+
+    @Test(expected = UnsupportedOperationException.class)
+    public void testOffHeapException()
+    {
+        long numKeys = (Integer.MAX_VALUE * 64) + 1; // approx 128 Billion
+        FilterFactory.getFilter(numKeys, 0.01d, true);
+    }
+
+    @Test
+    @Ignore
+    public void testHugeBFSerialization() throws IOException
+    {
+        ByteBuffer test = ByteBuffer.wrap(new byte[] {0, 1});
+
+        File file = FileUtils.createTempFile("bloomFilterTest-", ".dat");
+        BloomFilter filter = (BloomFilter) FilterFactory.getFilter(((long)Integer.MAX_VALUE
/ 8) + 1, 0.01d, true);
+        filter.add(test);
+        DataOutputStream out = new DataOutputStream(new FileOutputStream(file));
+        FilterFactory.serialize(filter, out);
+        filter.bitset.serialize(out);
+        out.close();
+        
+        DataInputStream in = new DataInputStream(new FileInputStream(file));
+        BloomFilter filter2 = (BloomFilter) FilterFactory.deserialize(in, Type.MURMUR3, true);
+        Assert.assertTrue(filter2.isPresent(test));
+        FileUtils.closeQuietly(in);
+    }
 }


Mime
View raw message