Return-Path: X-Original-To: apmail-cassandra-commits-archive@www.apache.org Delivered-To: apmail-cassandra-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 522A210D5D for ; Thu, 22 Aug 2013 02:25:08 +0000 (UTC) Received: (qmail 77462 invoked by uid 500); 22 Aug 2013 02:25:07 -0000 Delivered-To: apmail-cassandra-commits-archive@cassandra.apache.org Received: (qmail 77416 invoked by uid 500); 22 Aug 2013 02:25:07 -0000 Mailing-List: contact commits-help@cassandra.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cassandra.apache.org Delivered-To: mailing list commits@cassandra.apache.org Received: (qmail 77302 invoked by uid 99); 22 Aug 2013 02:25:07 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 22 Aug 2013 02:25:07 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 778C78C2553; Thu, 22 Aug 2013 02:25:07 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vijay@apache.org To: commits@cassandra.apache.org Date: Thu, 22 Aug 2013 02:25:07 -0000 Message-Id: <664a5432f83845f0b4068ac8092e5446@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/4] git commit: additional check for offheap bloom filter size patch by Vijay; reviewed by jbellis for CASSANDRA-5903 Updated Branches: refs/heads/trunk 5e33b9b03 -> 74492ef27 additional check for offheap bloom filter size patch by Vijay; reviewed by jbellis for CASSANDRA-5903 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9bb4d93e Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9bb4d93e Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9bb4d93e Branch: refs/heads/trunk Commit: 9bb4d93e3c8471012a39cbb0a1be5b082115a42b Parents: 4bc8c89 Author: Vijay Parthasarathy Authored: Wed Aug 21 19:21:28 2013 -0700 Committer: Vijay Parthasarathy Committed: Wed Aug 21 19:21:28 2013 -0700 ---------------------------------------------------------------------- .../cassandra/utils/obs/OffHeapBitSet.java | 5 ++- .../apache/cassandra/utils/BloomFilterTest.java | 36 ++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/9bb4d93e/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java b/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java index fb32043..c103835 100644 --- a/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java +++ b/src/java/org/apache/cassandra/utils/obs/OffHeapBitSet.java @@ -36,9 +36,12 @@ public class OffHeapBitSet implements IBitSet public OffHeapBitSet(long numBits) { // OpenBitSet.bits2words calculation is there for backward compatibility. - long byteCount = OpenBitSet.bits2words(numBits) * 8L; + long wordCount = OpenBitSet.bits2words(numBits); + if (wordCount > Integer.MAX_VALUE) + throw new UnsupportedOperationException("Bloom filter size is > 16GB, reduce the bloom_filter_fp_chance"); try { + long byteCount = wordCount * 8L; bytes = RefCountedMemory.allocate(byteCount); } catch (OutOfMemoryError e) http://git-wip-us.apache.org/repos/asf/cassandra/blob/9bb4d93e/test/unit/org/apache/cassandra/utils/BloomFilterTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java index bb0865f..562e3ea 100644 --- a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java +++ b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java @@ -20,16 +20,25 @@ package org.apache.cassandra.utils; import java.io.ByteArrayInputStream; import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.HashSet; import java.util.Iterator; import java.util.Set; +import junit.framework.Assert; + import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.utils.FilterFactory.Type; public class BloomFilterTest { @@ -138,4 +147,31 @@ public class BloomFilterTest { testManyHashes(FilterTestHelper.randomKeys()); } + + @Test(expected = UnsupportedOperationException.class) + public void testOffHeapException() + { + long numKeys = (Integer.MAX_VALUE * 64) + 1; // approx 128 Billion + FilterFactory.getFilter(numKeys, 0.01d, true); + } + + @Test + @Ignore + public void testHugeBFSerialization() throws IOException + { + ByteBuffer test = ByteBuffer.wrap(new byte[] {0, 1}); + + File file = FileUtils.createTempFile("bloomFilterTest-", ".dat"); + BloomFilter filter = (BloomFilter) FilterFactory.getFilter(((long)Integer.MAX_VALUE / 8) + 1, 0.01d, true); + filter.add(test); + DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); + FilterFactory.serialize(filter, out); + filter.bitset.serialize(out); + out.close(); + + DataInputStream in = new DataInputStream(new FileInputStream(file)); + BloomFilter filter2 = (BloomFilter) FilterFactory.deserialize(in, Type.MURMUR3, true); + Assert.assertTrue(filter2.isPresent(test)); + FileUtils.closeQuietly(in); + } }