db-derby-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From krist...@apache.org
Subject svn commit: r756551 - in /db/derby/code/trunk/java: engine/org/apache/derby/impl/io/vfmem/BlockedByteArray.java testing/org/apache/derbyTesting/unitTests/junit/BlockedByteArrayTest.java
Date Fri, 20 Mar 2009 16:01:54 GMT
Author: kristwaa
Date: Fri Mar 20 16:01:54 2009
New Revision: 756551

URL: http://svn.apache.org/viewvc?rev=756551&view=rev
Log:
DERBY-4103: Allow unlimited growth for in-memory conglomerates.
Allows a conglomerate in the in-memory back end to grow unlimited, that is
until the system runs out of memory. This patch removes the previous hard
limits set for conglomerates, which was dependent on the page size used.
Patch file: derby-4103-1a_unlimited_growth.diff


Modified:
    db/derby/code/trunk/java/engine/org/apache/derby/impl/io/vfmem/BlockedByteArray.java
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/unitTests/junit/BlockedByteArrayTest.java

Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/io/vfmem/BlockedByteArray.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/io/vfmem/BlockedByteArray.java?rev=756551&r1=756550&r2=756551&view=diff
==============================================================================
--- db/derby/code/trunk/java/engine/org/apache/derby/impl/io/vfmem/BlockedByteArray.java (original)
+++ db/derby/code/trunk/java/engine/org/apache/derby/impl/io/vfmem/BlockedByteArray.java Fri
Mar 20 16:01:54 2009
@@ -21,20 +21,13 @@
 
 package org.apache.derby.impl.io.vfmem;
 
+import org.apache.derby.shared.common.sanity.SanityManager;
+
 /**
  * Stores data in blocks, and supports reading/writing data from/into these
  * blocks.
  * <p>
  * The blocked array is expanded and shrunk as required.
- * <p>
- * The current implementation has a size limit of
- * {@code INITIAL_BLOCK_HOLDER_SIZE * blockSize}. For the default values, this
- * gives:
- * <ul> <li>4 KB blocks: 256 MB
- *      <li>8 KB blocks: 512 MB
- *      <li>16 KB blocks: 1024 MB
- *      <li>32 KB blocks: 2048 MB
- * </ul>
  */
 public class BlockedByteArray {
 
@@ -50,7 +43,12 @@
     private static final int DEFAULT_BLOCKSIZE = _4K;
 
     /** The default number of slots for holding a block of data. */
-    private static final int INITIAL_BLOCK_HOLDER_SIZE = 64*1024;
+    private static final int INITIAL_BLOCK_HOLDER_SIZE = 1024;
+    /**
+     * Minimum number of holder slots to grow with when the block holder array
+     * has to grow to be able to reference all the data arrays.
+     */
+    private static final int MIN_HOLDER_GROWTH = 1024;
 
     /** References to blocks of data. */
     private byte[][] blocks;
@@ -89,7 +87,7 @@
     }
 
     /**
-     * Reads the up to {@code len} bytes.
+     * Reads up to {@code len} bytes.
      *
      * @param pos the position to start reading at
      * @param buf the destination buffer
@@ -141,6 +139,10 @@
      * @param newLength the new length of the allocated data in bytes
      */
     public synchronized void setLength(final long newLength) {
+        // If capacity is requested before any writes has taken place.
+        if (blockSize == 0) {
+            checkBlockSize((int)Math.min(Integer.MAX_VALUE, newLength));
+        }
         final long currentCapacity = allocatedBlocks * blockSize;
         if (newLength > currentCapacity) {
             // Allocate more blocks.
@@ -157,6 +159,8 @@
                     blocks[i] = null;
                 }
                 allocatedBlocks = Math.min(allocatedBlocks, blocksToKeep);
+                // We keep the holder slots around, since the overhead for
+                // doing so is pretty small.
             }
         }
         length = Math.max(0L, newLength);
@@ -293,6 +297,9 @@
      */
     //@GuardedBy("this")
     private void increaseCapacity(long lastIndex) {
+        if (SanityManager.DEBUG) {
+            SanityManager.ASSERT(blockSize > 0, "Invalid/unset block size");
+        }
         // Safe-guard to avoid overwriting existing data.
         if (lastIndex < allocatedBlocks * blockSize) {
             return;
@@ -301,13 +308,20 @@
         // We may allocate one more array than required.
         final int blocksRequired = (int)((lastIndex) / blockSize) +1;
         if (blocksRequired > blocks.length) {
-            // TODO: Thrown an OOME or do something else here?
-            //       If we let the array grow unbounded, the JVM would throw
-            //       the OOME when get into the situation that all the
-            //       available memory is exhausted.
-            throw new IllegalStateException("Too big: ~" +
-                    ((lastIndex) / 1024 / 1024) + " MB");
+            // Grow the block holder array.
+            // Make sure we have enough slots. Note that we only grow the block
+            // holder array, we don't fill it with data blocks before needed.
+            int growTo = Math.max(
+                    // Grow at least ~33%.
+                    blocks.length + (blocks.length / 3),
+                    // For cases where we need to grow more than 33%.
+                    blocksRequired + MIN_HOLDER_GROWTH);
+            byte[][] tmpBlocks = blocks;
+            blocks = new byte[growTo][];
+            // Copy the data array references.
+            System.arraycopy(tmpBlocks, 0, blocks, 0, allocatedBlocks);
         }
+        // Allocate new data arrays to accomodate lastIndex bytes.
         for (int i=allocatedBlocks; i < blocksRequired; i++) {
             blocks[i] = new byte[blockSize];
         }

Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/unitTests/junit/BlockedByteArrayTest.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/unitTests/junit/BlockedByteArrayTest.java?rev=756551&r1=756550&r2=756551&view=diff
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/unitTests/junit/BlockedByteArrayTest.java
(original)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/unitTests/junit/BlockedByteArrayTest.java
Fri Mar 20 16:01:54 2009
@@ -136,6 +136,31 @@
         assertEquals(44444, src.length());
     }
 
+    /**
+     * Performs a series of capacity changes.
+     *
+     * @throws IOException if something goes wrong
+     */
+    public void testCapacityGrowth()
+            throws IOException {
+        BlockedByteArray src = createBlockedByteArray(0);
+        src.setLength(1*1024*1024); // 1 MB
+        src.setLength(10*1024*1024); // 10 MB
+        src.setLength(5*1024*1024); // 5 MB
+        src.setLength(7*1024*1024); // 7 MB
+        assertEquals(7*1024*1024L, src.length());
+        src.setLength(75*1024*1024); // 75 MB
+        src.setLength(0); // 0 bytes
+        assertEquals(0L, src.length());
+        src.setLength(39*1024*1024); // 39 MB
+        src.setLength(39*1024*1024+1); // 39 MB +1 B
+        assertEquals(39*1024*1024+1L, src.length());
+        src.setLength(39*1024*1024); // 39 MB
+        src.setLength(39*1024*1024); // 39 MB
+        src.setLength(-1); // Invalid value - causes array to be truncated.
+        assertEquals(0L, src.length());
+    }
+
     public static Test suite() {
         return new TestSuite(BlockedByteArrayTest.class);
     }



Mime
View raw message