accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [accumulo] branch 1.9 updated: Fix #1520 Limit UnsynchronizedBuffer max size (#1523)
Date Wed, 19 Feb 2020 23:25:54 GMT
This is an automated email from the ASF dual-hosted git repository.

ctubbsii pushed a commit to branch 1.9
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/1.9 by this push:
     new b67d3f9  Fix #1520 Limit UnsynchronizedBuffer max size (#1523)
b67d3f9 is described below

commit b67d3f93eea742523a98af95ca3a7aaceda5a990
Author: Christopher Tubbs <ctubbsii@apache.org>
AuthorDate: Wed Feb 19 18:25:44 2020 -0500

    Fix #1520 Limit UnsynchronizedBuffer max size (#1523)
    
    Limit UnsynchronizedBuffer maximum size to Integer.MAX_VALUE - 8, the
    same limit as ArrayList, because of some JVM behavior being unable to
    allocate arrays with Integer.MAX_VALUE.
---
 .../accumulo/core/util/UnsynchronizedBuffer.java   |  7 ++++--
 .../accumulo/core/file/rfile/RelativeKeyTest.java  | 24 ------------------
 .../core/util/UnsynchronizedBufferTest.java        | 29 ++++++++++++++++++++++
 3 files changed, 34 insertions(+), 26 deletions(-)

diff --git a/core/src/main/java/org/apache/accumulo/core/util/UnsynchronizedBuffer.java b/core/src/main/java/org/apache/accumulo/core/util/UnsynchronizedBuffer.java
index 2993f8b..8d84127 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/UnsynchronizedBuffer.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/UnsynchronizedBuffer.java
@@ -262,8 +262,11 @@ public class UnsynchronizedBuffer {
     if (i < 0)
       throw new IllegalArgumentException();
 
-    if (i > (1 << 30))
-      return Integer.MAX_VALUE; // this is the next power of 2 minus one... a special case
+    if (i > (1 << 30)) {
+      // this is the next power of 2 minus 8... a special case taken from ArrayList limits
+      // because some JVMs can't allocate an array that large
+      return Integer.MAX_VALUE - 8;
+    }
 
     if (i == 0) {
       return 1;
diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/RelativeKeyTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/RelativeKeyTest.java
index 87c606c..fd6ee15 100644
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RelativeKeyTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RelativeKeyTest.java
@@ -32,7 +32,6 @@ import org.apache.accumulo.core.data.PartialKey;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.file.rfile.RelativeKey.SkippR;
 import org.apache.accumulo.core.util.MutableByteSequence;
-import org.apache.accumulo.core.util.UnsynchronizedBuffer;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -40,29 +39,6 @@ import org.junit.Test;
 public class RelativeKeyTest {
 
   @Test
-  public void testBasicRelativeKey() {
-    assertEquals(1, UnsynchronizedBuffer.nextArraySize(0));
-    assertEquals(1, UnsynchronizedBuffer.nextArraySize(1));
-    assertEquals(2, UnsynchronizedBuffer.nextArraySize(2));
-    assertEquals(4, UnsynchronizedBuffer.nextArraySize(3));
-    assertEquals(4, UnsynchronizedBuffer.nextArraySize(4));
-    assertEquals(8, UnsynchronizedBuffer.nextArraySize(5));
-    assertEquals(8, UnsynchronizedBuffer.nextArraySize(8));
-    assertEquals(16, UnsynchronizedBuffer.nextArraySize(9));
-
-    assertEquals(1 << 16, UnsynchronizedBuffer.nextArraySize((1 << 16) - 1));
-    assertEquals(1 << 16, UnsynchronizedBuffer.nextArraySize(1 << 16));
-    assertEquals(1 << 17, UnsynchronizedBuffer.nextArraySize((1 << 16) + 1));
-
-    assertEquals(1 << 30, UnsynchronizedBuffer.nextArraySize((1 << 30) - 1));
-
-    assertEquals(1 << 30, UnsynchronizedBuffer.nextArraySize(1 << 30));
-
-    assertEquals(Integer.MAX_VALUE, UnsynchronizedBuffer.nextArraySize(Integer.MAX_VALUE
- 1));
-    assertEquals(Integer.MAX_VALUE, UnsynchronizedBuffer.nextArraySize(Integer.MAX_VALUE));
-  }
-
-  @Test
   public void testCommonPrefix() {
     // exact matches
     ArrayByteSequence exact = new ArrayByteSequence("abc");
diff --git a/core/src/test/java/org/apache/accumulo/core/util/UnsynchronizedBufferTest.java
b/core/src/test/java/org/apache/accumulo/core/util/UnsynchronizedBufferTest.java
index 0038b06..bc68ba2 100644
--- a/core/src/test/java/org/apache/accumulo/core/util/UnsynchronizedBufferTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/UnsynchronizedBufferTest.java
@@ -125,4 +125,33 @@ public class UnsynchronizedBufferTest {
     assertTrue("The byte array written to by UnsynchronizedBuffer is not equal to WritableUtils",
         Arrays.equals(hadoopBytes, accumuloBytes));
   }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testNextArraySizeNegative() {
+    UnsynchronizedBuffer.nextArraySize(-1);
+  }
+
+  @Test
+  public void testNextArraySize() {
+    // 0 <= size <= 2^0
+    assertEquals(1, UnsynchronizedBuffer.nextArraySize(0));
+    assertEquals(1, UnsynchronizedBuffer.nextArraySize(1));
+
+    // 2^0 < size <= 2^1
+    assertEquals(2, UnsynchronizedBuffer.nextArraySize(2));
+
+    // 2^1 < size <= 2^30
+    for (int exp = 1; exp < 30; ++exp) {
+      // 2^exp < size <= 2^(exp+1) (for all exp: [1,29])
+      int nextExp = exp + 1;
+      assertEquals(1 << nextExp, UnsynchronizedBuffer.nextArraySize((1 << exp)
+ 1));
+      assertEquals(1 << nextExp, UnsynchronizedBuffer.nextArraySize(1 << nextExp));
+    }
+    // 2^30 < size < Integer.MAX_VALUE
+    assertEquals(Integer.MAX_VALUE - 8, UnsynchronizedBuffer.nextArraySize((1 << 30)
+ 1));
+    assertEquals(Integer.MAX_VALUE - 8, UnsynchronizedBuffer.nextArraySize(Integer.MAX_VALUE
- 9));
+    assertEquals(Integer.MAX_VALUE - 8, UnsynchronizedBuffer.nextArraySize(Integer.MAX_VALUE
- 8));
+    assertEquals(Integer.MAX_VALUE - 8, UnsynchronizedBuffer.nextArraySize(Integer.MAX_VALUE));
+  }
+
 }


Mime
View raw message