db-derby-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From krist...@apache.org
Subject svn commit: r756612 - in /db/derby/code/branches/10.5/java: engine/org/apache/derby/impl/io/vfmem/BlockedByteArray.java testing/org/apache/derbyTesting/unitTests/junit/BlockedByteArrayTest.java
Date Fri, 20 Mar 2009 16:47:25 GMT
Author: kristwaa
Date: Fri Mar 20 16:47:24 2009
New Revision: 756612

URL: http://svn.apache.org/viewvc?rev=756612&view=rev
Log:
DERBY-4103: Allow unlimited growth for in-memory conglomerates.
Merged revision 756551 from trunk.

Modified:
    db/derby/code/branches/10.5/java/engine/org/apache/derby/impl/io/vfmem/BlockedByteArray.java
    db/derby/code/branches/10.5/java/testing/org/apache/derbyTesting/unitTests/junit/BlockedByteArrayTest.java

Modified: db/derby/code/branches/10.5/java/engine/org/apache/derby/impl/io/vfmem/BlockedByteArray.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.5/java/engine/org/apache/derby/impl/io/vfmem/BlockedByteArray.java?rev=756612&r1=756611&r2=756612&view=diff
==============================================================================
--- db/derby/code/branches/10.5/java/engine/org/apache/derby/impl/io/vfmem/BlockedByteArray.java
(original)
+++ db/derby/code/branches/10.5/java/engine/org/apache/derby/impl/io/vfmem/BlockedByteArray.java
Fri Mar 20 16:47:24 2009
@@ -21,20 +21,13 @@
 
 package org.apache.derby.impl.io.vfmem;
 
+import org.apache.derby.shared.common.sanity.SanityManager;
+
 /**
  * Stores data in blocks, and supports reading/writing data from/into these
  * blocks.
  * <p>
  * The blocked array is expanded and shrunk as required.
- * <p>
- * The current implementation has a size limit of
- * {@code INITIAL_BLOCK_HOLDER_SIZE * blockSize}. For the default values, this
- * gives:
- * <ul> <li>4 KB blocks: 256 MB
- *      <li>8 KB blocks: 512 MB
- *      <li>16 KB blocks: 1024 MB
- *      <li>32 KB blocks: 2048 MB
- * </ul>
  */
 public class BlockedByteArray {
 
@@ -50,7 +43,12 @@
     private static final int DEFAULT_BLOCKSIZE = _4K;
 
     /** The default number of slots for holding a block of data. */
-    private static final int INITIAL_BLOCK_HOLDER_SIZE = 64*1024;
+    private static final int INITIAL_BLOCK_HOLDER_SIZE = 1024;
+    /**
+     * Minimum number of holder slots to grow with when the block holder array
+     * has to grow to be able to reference all the data arrays.
+     */
+    private static final int MIN_HOLDER_GROWTH = 1024;
 
     /** References to blocks of data. */
     private byte[][] blocks;
@@ -89,7 +87,7 @@
     }
 
     /**
-     * Reads the up to {@code len} bytes.
+     * Reads up to {@code len} bytes.
      *
      * @param pos the position to start reading at
      * @param buf the destination buffer
@@ -141,6 +139,10 @@
      * @param newLength the new length of the allocated data in bytes
      */
     public synchronized void setLength(final long newLength) {
+        // If capacity is requested before any writes has taken place.
+        if (blockSize == 0) {
+            checkBlockSize((int)Math.min(Integer.MAX_VALUE, newLength));
+        }
         final long currentCapacity = allocatedBlocks * blockSize;
         if (newLength > currentCapacity) {
             // Allocate more blocks.
@@ -157,6 +159,8 @@
                     blocks[i] = null;
                 }
                 allocatedBlocks = Math.min(allocatedBlocks, blocksToKeep);
+                // We keep the holder slots around, since the overhead for
+                // doing so is pretty small.
             }
         }
         length = Math.max(0L, newLength);
@@ -293,6 +297,9 @@
      */
     //@GuardedBy("this")
     private void increaseCapacity(long lastIndex) {
+        if (SanityManager.DEBUG) {
+            SanityManager.ASSERT(blockSize > 0, "Invalid/unset block size");
+        }
         // Safe-guard to avoid overwriting existing data.
         if (lastIndex < allocatedBlocks * blockSize) {
             return;
@@ -301,13 +308,20 @@
         // We may allocate one more array than required.
         final int blocksRequired = (int)((lastIndex) / blockSize) +1;
         if (blocksRequired > blocks.length) {
-            // TODO: Thrown an OOME or do something else here?
-            //       If we let the array grow unbounded, the JVM would throw
-            //       the OOME when get into the situation that all the
-            //       available memory is exhausted.
-            throw new IllegalStateException("Too big: ~" +
-                    ((lastIndex) / 1024 / 1024) + " MB");
+            // Grow the block holder array.
+            // Make sure we have enough slots. Note that we only grow the block
+            // holder array, we don't fill it with data blocks before needed.
+            int growTo = Math.max(
+                    // Grow at least ~33%.
+                    blocks.length + (blocks.length / 3),
+                    // For cases where we need to grow more than 33%.
+                    blocksRequired + MIN_HOLDER_GROWTH);
+            byte[][] tmpBlocks = blocks;
+            blocks = new byte[growTo][];
+            // Copy the data array references.
+            System.arraycopy(tmpBlocks, 0, blocks, 0, allocatedBlocks);
         }
+        // Allocate new data arrays to accomodate lastIndex bytes.
         for (int i=allocatedBlocks; i < blocksRequired; i++) {
             blocks[i] = new byte[blockSize];
         }

Modified: db/derby/code/branches/10.5/java/testing/org/apache/derbyTesting/unitTests/junit/BlockedByteArrayTest.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.5/java/testing/org/apache/derbyTesting/unitTests/junit/BlockedByteArrayTest.java?rev=756612&r1=756611&r2=756612&view=diff
==============================================================================
--- db/derby/code/branches/10.5/java/testing/org/apache/derbyTesting/unitTests/junit/BlockedByteArrayTest.java
(original)
+++ db/derby/code/branches/10.5/java/testing/org/apache/derbyTesting/unitTests/junit/BlockedByteArrayTest.java
Fri Mar 20 16:47:24 2009
@@ -136,6 +136,31 @@
         assertEquals(44444, src.length());
     }
 
+    /**
+     * Performs a series of capacity changes.
+     *
+     * @throws IOException if something goes wrong
+     */
+    public void testCapacityGrowth()
+            throws IOException {
+        BlockedByteArray src = createBlockedByteArray(0);
+        src.setLength(1*1024*1024); // 1 MB
+        src.setLength(10*1024*1024); // 10 MB
+        src.setLength(5*1024*1024); // 5 MB
+        src.setLength(7*1024*1024); // 7 MB
+        assertEquals(7*1024*1024L, src.length());
+        src.setLength(75*1024*1024); // 75 MB
+        src.setLength(0); // 0 bytes
+        assertEquals(0L, src.length());
+        src.setLength(39*1024*1024); // 39 MB
+        src.setLength(39*1024*1024+1); // 39 MB +1 B
+        assertEquals(39*1024*1024+1L, src.length());
+        src.setLength(39*1024*1024); // 39 MB
+        src.setLength(39*1024*1024); // 39 MB
+        src.setLength(-1); // Invalid value - causes array to be truncated.
+        assertEquals(0L, src.length());
+    }
+
     public static Test suite() {
         return new TestSuite(BlockedByteArrayTest.class);
     }



Mime
View raw message