Mailing-List: contact derby-commits-help@db.apache.org; run by ezmlm
Precedence: bulk
Reply-To: "Derby Development" <derby-dev@db.apache.org>
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r779849 - in /db/derby/code/branches/10.5/java:
 engine/org/apache/derby/impl/store/raw/data/BasePage.java
 testing/org/apache/derbyTesting/functionTests/tests/store/ClobReclamationTest.java
Date: Fri, 29 May 2009 05:53:08 -0000
To: derby-commits@db.apache.org
From: mikem@apache.org
Message-Id: <20090529055308.937D32388873@eris.apache.org>

Author: mikem
Date: Fri May 29 05:53:08 2009
New Revision: 779849

URL: http://svn.apache.org/viewvc?rev=779849&view=rev
Log:
DERBY-4182
backporting svn 778926 from trunk to 10.5 branch.

Before this fix abort of inserts that included clob or blob chains would
destroy the links of the allocated pages of the chains. This would leave
allocated pages that could never be reclaimed either by subsequent post
commit processing or inplace compress. Only offline compress could reclaim
the space. This fix changes insert abort processing to automatically put
all pieces of long columns except for the head page on the free list as part
of the abort.

Note this does not fix existing tables that have had this problem happen in
the past, only stops it from happening. One must run an offline compress to
reclaim this space to fix any instances of this bug prior to this fix. 


Modified:
    db/derby/code/branches/10.5/java/engine/org/apache/derby/impl/store/raw/data/BasePage.java
    db/derby/code/branches/10.5/java/testing/org/apache/derbyTesting/functionTests/tests/store/ClobReclamationTest.java

Modified: db/derby/code/branches/10.5/java/engine/org/apache/derby/impl/store/raw/data/BasePage.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.5/java/engine/org/apache/derby/impl/store/raw/data/BasePage.java?rev=779849&r1=779848&r2=779849&view=diff
==============================================================================
--- db/derby/code/branches/10.5/java/engine/org/apache/derby/impl/store/raw/data/BasePage.java (original)
+++ db/derby/code/branches/10.5/java/engine/org/apache/derby/impl/store/raw/data/BasePage.java Fri May 29 05:53:08 2009
@@ -815,31 +815,93 @@
 
 	}
 
-	/**
-	  
-		When we update a column, it turned into a long column.  Need to change
-		the update to effectively insert a new long column chain.
-
-		@exception StandardException Unexpected exception from the implementation
-	 */
-	protected RecordHandle insertLongColumn(BasePage mainChainPage,
-			LongColumnException lce, byte insertFlag)
+    /**
+     * Routine to insert a long column.
+     * <p>
+     * This code inserts a long column as a linked list of rows on overflow
+     * pages.  This list is pointed to by a small pointer in the main page
+     * row column.  The operation does the following:
+     *     allocate new overflow page
+     *     insert single row filling overflow page
+     *     while (more of column exists)
+     *         allocate new overflow page
+     *         insert single row with next piece of row
+     *         update previous piece to point to this new piece of row
+     *
+     * Same code is called both from an initial insert of a long column and
+     * from a subsequent update that results in a long column.
+     *
+     * @return The recordHandle of the first piece of the long column chain.
+     *
+     * @param mainChainPage The parent page with row piece containing column
+     *                      that will eventually point to this long column
+     *                      chain.
+     * @param lce           The LongColumnException thrown when we recognized
+     *                      that the column being inserted was "long", this 
+     *                      structure is used to cache the info that we have
+     *                      read so far about column.  In the case of an insert
+     *                      of the stream it will have a copy of just the first
+     *                      page of the stream that has already been read once.
+     * @param insertFlag    flags for insert operation.    
+     *
+     *
+     * @exception  StandardException  Standard exception policy.
+     **/
+	protected RecordHandle insertLongColumn(
+    BasePage            mainChainPage,
+    LongColumnException lce, 
+    byte                insertFlag)
 		throws StandardException
 	{
 
-		// Object[] row = new Object[1];
-		// row[0] = (Object) lce.getColumn();
 		Object[] row = new Object[1];
-		row[0] = lce.getColumn();
+		row[0]       = lce.getColumn();
 
 		RecordHandle firstHandle = null;
-		RecordHandle handle = null;
-		RecordHandle prevHandle = null;
-		BasePage curPage = mainChainPage;
-		BasePage prevPage = null;
-		boolean isFirstPage = true;
+		RecordHandle handle      = null;
+		RecordHandle prevHandle  = null;
+		BasePage     curPage     = mainChainPage;
+		BasePage     prevPage    = null;
+		boolean      isFirstPage = true;
+        
+        // undo inserts as purges of all pieces of the overflow column
+        // except for the 1st overflow page pointed at by the main row.  
+        //
+        // Consider a row with one column which is a long column
+        // that takes 2 pages for itself plus an entry in main parent page.
+        // the log records in order for this look something like:
+        //     insert overflow page 1
+        //     insert overflow page 2
+        //     update overflow page 1 record to have pointer to overflow page 2
+        //     insert main row (which has pointer to overflow page 1)
+        //
+        // If this insert gets aborted then something like the following 
+        // happens:
+        //     main row is marked deleted (but ptr to overflow 1 still exists)
+        //     update is aborted so link on page 2 to page 1 is lost
+        //     overflow row on page 2 is marked deleted
+        //     overflow row on page 1 is marked deleted
+        //
+        // There is no way to reclaim page 2 later as the abort of the update
+        // has now lost the link from overflow page 1 to overflow 2, so 
+        // the system has to do it as part of the abort of the insert.  But, 
+        // it can't for page 1 as the main page will attempt to follow
+        // it's link in the deleted row during it's space reclamation and it 
+        // can't tell the difference 
+        // between a row that has been marked deleted as part of an aborted 
+        // insert or as part of a committed delete.  When it follows the link
+        // it could find no page and that could be coded against, but it could 
+        // be that the page is now used by some other overflow row which would 
+        // lead to lots of different kinds of problems.
+        //
+        // So the code leaves the 1st overflow page to be cleaned up with the
+        // main page row is purged, but goes ahead and immediately purges all
+        // the segments that will be lost as part of the links being lost due
+        // to aborted updates.
+        byte after_first_page_insertFlag = 
+            (byte) (insertFlag | Page.INSERT_UNDO_WITH_PURGE);
 
-		// when inserting a long column startCOlumn is just used
+		// when inserting a long column startColumn is just used
 		// as a flag. -1 means the insert is complete, != -1 indicates
 		// more inserts are required.
 		int startColumn = 0;
@@ -873,9 +935,13 @@
 				firstHandle = handle;
 
 			// step 2: insert column portion
-			startColumn = owner.getActionSet().actionInsert(t, curPage, slot, recordId,
-				row, (FormatableBitSet)null, (LogicalUndo) null, insertFlag,
-				startColumn, true, -1, (DynamicByteArrayOutputStream) null, -1, 100);
+			startColumn = 
+                owner.getActionSet().actionInsert(
+                    t, curPage, slot, recordId, row, (FormatableBitSet)null, 
+                    (LogicalUndo) null, 
+                    (isFirstPage ? insertFlag : after_first_page_insertFlag), 
+                    startColumn, true, -1, (DynamicByteArrayOutputStream) null, 
+                    -1, 100);
 
 			// step 3: if it is not the first page, update previous page,
 			// then release latch on prevPage
@@ -885,8 +951,9 @@
 				prevPage.updateFieldOverflowDetails(prevHandle, handle);
 				prevPage.unlatch();
 				prevPage = null;
-			} else
+			} else {
 				isFirstPage = false;
+            }
 
 		} while (startColumn != (-1)) ;
 
@@ -1188,7 +1255,7 @@
 			// Before we purge these rows, we need to make sure they don't have
 			// overflow rows and columns.  Only clean up long rows and long
 			// columns if this is not a temporary container, otherwise, just
-			// loose the space.
+			// lose the space.
 
 			if (owner.isTemporaryContainer() ||	entireRecordOnPage(slot+i))
 				continue;

Modified: db/derby/code/branches/10.5/java/testing/org/apache/derbyTesting/functionTests/tests/store/ClobReclamationTest.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.5/java/testing/org/apache/derbyTesting/functionTests/tests/store/ClobReclamationTest.java?rev=779849&r1=779848&r2=779849&view=diff
==============================================================================
--- db/derby/code/branches/10.5/java/testing/org/apache/derbyTesting/functionTests/tests/store/ClobReclamationTest.java (original)
+++ db/derby/code/branches/10.5/java/testing/org/apache/derbyTesting/functionTests/tests/store/ClobReclamationTest.java Fri May 29 05:53:08 2009
@@ -19,11 +19,13 @@
  */
 package org.apache.derbyTesting.functionTests.tests.store;
 
+import java.sql.CallableStatement;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
+
 import java.util.Properties;
 
 import junit.framework.Test;
@@ -119,6 +121,32 @@
                 + expectedAlloc } });
     }
 
+    /**
+     * Check that table has specified number of free pages.
+     *
+     * @param table
+     * @param expectedFree  expected number of free pages.
+     * @throws SQLException
+     */
+    private void checkNumFreePages(
+    String  table, 
+    int     expectedFree) throws SQLException {        
+
+        // Check the space table 
+        // Should not have grown.
+
+        PreparedStatement ps = 
+            prepareStatement(
+                  "SELECT NUMFREEPAGES FROM "
+                + " new org.apache.derby.diag.SpaceTable('APP',?) t"
+                + " WHERE CONGLOMERATENAME = ?");
+
+        ps.setString(1, table);
+        ps.setString(2, table);
+        ResultSet rs = ps.executeQuery();
+        JDBC.assertFullResultSet(rs, new String[][] { { "" + expectedFree } });
+    }
+
     private static void fiveHundredUpdates(Connection conn,
             String updateString, int key, boolean lockTable) throws SQLException {
         PreparedStatement ps = conn
@@ -185,6 +213,72 @@
         }
         checkNumAllocatedPages("CLOBTAB2",1);
     }
+
+    /**
+     * Test for DERBY-4182.  
+     *
+     * This test just exercises the abort specific part of DERBY-4182.  After
+     * the fix abort of an insert containing a blob will leave the head row, 
+     * plus the first page of the overflow chain.  The rest of the chain
+     * will be moved to free pages.
+     *
+     * @throws SQLException
+     */
+    public void testBlobLinkedListReclamationOnRollback() throws SQLException {
+        setAutoCommit(false);
+
+        int clob_length = 200000;
+
+        // pick a clob bigger than 2*max page size
+        String insertString = Formatters.repeatChar("a", 200000);
+        PreparedStatement ps = 
+            prepareStatement("INSERT INTO CLOBTAB3 VALUES(?,?)");
+
+        int numrows = 500;
+
+        for (int i = 0; i < numrows; i++) {            
+            ps.setInt(1, i);
+            ps.setString(2, insertString);   
+            ps.executeUpdate();
+            rollback();
+        }
+        ps.close();
+
+        // until DERBY-4057 fixed expect space to be 2 pages per row plus
+        // 1 head page per container.
+        checkNumAllocatedPages("CLOBTAB3", (numrows * 2) + 1);
+
+        // expect most free pages to get used by subsequent inserts.  Only 
+        // free pages should be from the last remaining aborted insert of
+        // the last clob chain.  It should include all of the clob except the
+        // head page of the chain: (sizeof(clob) / page size) - 1
+        // Derby should default to 32k page size for any table with a clob in
+        // it.
+ 
+        // (clob length / page size ) + 
+        //     1 page for int divide round off - 1 for the head page.
+        checkNumFreePages("CLOBTAB3", (clob_length / 32000) + 1 - 1);
+        commit();
+
+        // running inplace compress should reclaim all the remaining aborted
+        // insert space, previous to fix inplace compress would leave stranded
+        // allocated pages that were part of the clob overflow chains.
+        CallableStatement call_compress =
+            prepareCall(
+                "CALL SYSCS_UTIL.SYSCS_INPLACE_COMPRESS_TABLE(?, ?, 1, 1, 1)");
+
+        call_compress.setString(1, "APP");
+        call_compress.setString(2, "CLOBTAB3");
+        call_compress.executeUpdate();
+
+        // all space except for head page should be reclaimed.
+        checkNumAllocatedPages("CLOBTAB3", 1);
+
+        // should be no free pages after the inplace compress of a table with
+        // all deleted rows.
+        checkNumFreePages("CLOBTAB3", 0);
+        commit();
+    }
     
     public static Test suite() {
 
@@ -210,6 +304,7 @@
                     ps.executeUpdate();
                 }
                 s.executeUpdate("CREATE TABLE CLOBTAB2 (I INT, C CLOB)");                
+                s.executeUpdate("CREATE TABLE CLOBTAB3 (I INT, C CLOB)");                
             }
 
         };