Author: kmarsden Date: Thu Feb 12 16:40:16 2009 New Revision: 743798 URL: http://svn.apache.org/viewvc?rev=743798&view=rev Log: DERBY-4050 Multithreaded clob update causes growth in table that does not get reclaimed merge 743023 from trunk. Rework test to run in old harness. Added: db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/master/ClobReclamationTest.out (with props) db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/tests/store/ClobReclamationTest.java (with props) db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/tests/store/ClobReclamationTest_derby.properties (with props) Modified: db/derby/code/branches/10.2/java/engine/org/apache/derby/impl/store/raw/data/ReclaimSpaceHelper.java db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/suites/storemore.runall Modified: db/derby/code/branches/10.2/java/engine/org/apache/derby/impl/store/raw/data/ReclaimSpaceHelper.java URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.2/java/engine/org/apache/derby/impl/store/raw/data/ReclaimSpaceHelper.java?rev=743798&r1=743797&r2=743798&view=diff ============================================================================== --- db/derby/code/branches/10.2/java/engine/org/apache/derby/impl/store/raw/data/ReclaimSpaceHelper.java (original) +++ db/derby/code/branches/10.2/java/engine/org/apache/derby/impl/store/raw/data/ReclaimSpaceHelper.java Thu Feb 12 16:40:16 2009 @@ -259,9 +259,28 @@ } if (work.incrAttempts() < 3) // retry this for serveral times + { return Serviceable.REQUEUE; + } else + { + // If code gets here, the space will be lost forever, and + // can only be reclaimed by a full offline compress of the + // table/index. + + if (SanityManager.DEBUG) + { + if (SanityManager.DEBUG_ON(DaemonService.DaemonTrace)) + { + SanityManager.DEBUG( + DaemonService.DaemonTrace, + " gave up after 3 tries to get container lock " + + work); + } + } + return Serviceable.DONE; + } } // At this point, container is opened with IX lock. @@ -281,8 +300,8 @@ return Serviceable.DONE; } - // We are reclaiming row space or long column. First get an xlock on the - // head row piece. + // We are reclaiming row space or long column. + // First get an xlock on the head row piece. RecordHandle headRecord = work.getHeadRowHandle(); if (!container_rlock.lockRecordForWrite( @@ -291,9 +310,27 @@ // cannot get the row lock, retry tran.abort(); if (work.incrAttempts() < 3) + { return Serviceable.REQUEUE; + } else + { + // If code gets here, the space will be lost forever, and + // can only be reclaimed by a full offline compress of the + // table/index. + + if (SanityManager.DEBUG) + { + if (SanityManager.DEBUG_ON(DaemonService.DaemonTrace)) + { + SanityManager.DEBUG( + DaemonService.DaemonTrace, + " gave up after 3 tries to get row lock " + + work); + } + } return Serviceable.DONE; + } } // The exclusive lock on the head row has been gotten. @@ -332,19 +369,33 @@ // operation. // long headPageId = ((PageKey)headRecord.getPageId()).getPageNumber(); + //DERBY-4050 - we wait for the page so we don't have to retry. + // prior to the 4050 fix, we called getPageNoWait and just + // retried 3 times. This left unreclaimed space if we were + // not successful after three tries. StoredPage headRowPage = - (StoredPage)containerHdl.getPageNoWait(headPageId); - + (StoredPage)containerHdl.getPage(headPageId); if (headRowPage == null) { - // Cannot get page no wait, try again later. + // It is not clear why headRowPage would be null, + // but logging the failure in case it happens. + // If code gets here, the space will be lost forever, and + // can only be reclaimed by a full offline compress of the + // table/index. + + if (SanityManager.DEBUG) + { + if (SanityManager.DEBUG_ON(DaemonService.DaemonTrace)) + { + SanityManager.DEBUG( + DaemonService.DaemonTrace, + "gave up because hadRowPage was null" + + work); + } + } tran.abort(); - if (work.incrAttempts() < 3) - return Serviceable.REQUEUE; - else - return Serviceable.DONE; + return Serviceable.DONE; } - try { headRowPage.removeOrphanedColumnChain(work, containerHdl); Added: db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/master/ClobReclamationTest.out URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/master/ClobReclamationTest.out?rev=743798&view=auto ============================================================================== --- db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/master/ClobReclamationTest.out (added) +++ db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/master/ClobReclamationTest.out Thu Feb 12 16:40:16 2009 @@ -0,0 +1 @@ +PASS: NUMALLOCATEDPAGES =5 Propchange: db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/master/ClobReclamationTest.out ------------------------------------------------------------------------------ svn:eol-style = native Modified: db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/suites/storemore.runall URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/suites/storemore.runall?rev=743798&r1=743797&r2=743798&view=diff ============================================================================== --- db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/suites/storemore.runall (original) +++ db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/suites/storemore.runall Thu Feb 12 16:40:16 2009 @@ -36,3 +36,4 @@ store/BackupPathTests.java store/LogDeviceTest.java store/BootAllTest.junit +store/ClobReclamationTest \ No newline at end of file Added: db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/tests/store/ClobReclamationTest.java URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/tests/store/ClobReclamationTest.java?rev=743798&view=auto ============================================================================== --- db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/tests/store/ClobReclamationTest.java (added) +++ db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/tests/store/ClobReclamationTest.java Thu Feb 12 16:40:16 2009 @@ -0,0 +1,125 @@ +/* + * + * Derby - Class org.apache.derbyTesting.functionTests.tests.store.ClobReclamationTest + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ +package org.apache.derbyTesting.functionTests.tests.store; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import org.apache.derby.tools.ij; +import org.apache.derbyTesting.functionTests.util.Formatters; + + + +/** + * Verify that space gets reclaimed for multi-threaded Clob updates + * + */ +public class ClobReclamationTest { + + // Need to adjust NUM_THREADS and expectedNumAllocated. + // For 2 threads expectedNumAllocated is 5 + // For 100 threads expectedNumAllocated is 201 + private static final int NUM_THREADS = 2; + + private static final int expectedNumAllocated = 5; + + + /** + * Two threads simultaneously updating a table. Thread 1 updates row 1 with + * a long value (>32K) Thread 2 updates row with a short clob ("hello"); + * NUMALLOCATEDPAGES should be only 3 after each does 500 updates + * + * @throws SQLException + * @throws InterruptedException + */ + public static void testMultiThreadedUpdate(Connection conn) throws SQLException, + InterruptedException { + + final String updateString = Formatters.repeatChar("a",33000); + Thread[] threads = new Thread[NUM_THREADS]; + for (int i = 0; i < NUM_THREADS; i++) { + final int key = i + 1; + threads[i] = new Thread() { + public void run() { + try { + Connection conn = ij.startJBMS(); + ClobReclamationTest.fiveHundredUpdates(conn, + updateString, key); + } catch (Exception e) { + e.printStackTrace(); + } + } + }; + } + for (int i = 0; i < NUM_THREADS; i++) { + threads[i].start(); + } + for (int i = 0; i < NUM_THREADS; i++) { + threads[i].join(); + } + + Statement s = conn.createStatement(); + // Check the space table + // Should not have grown. + ResultSet rs = s.executeQuery("SELECT NUMALLOCATEDPAGES FROM " + + " new org.apache.derby.diag.SpaceTable('APP','CLOBTAB') t" + + " WHERE CONGLOMERATENAME = 'CLOBTAB'"); + rs.next(); + int numAllocated = rs.getInt(1); + if (numAllocated == expectedNumAllocated) + System.out.println("PASS: NUMALLOCATEDPAGES =" + numAllocated); + else + System.out.println("FAIL: NUMALLOCATEDPAGES =" + numAllocated); + + } + + private static void fiveHundredUpdates(Connection conn, + String updateString, int key) throws SQLException { + PreparedStatement ps = conn + .prepareStatement("UPDATE CLOBTAB SET C = ? WHERE I = ?"); + for (int i = 0; i < 500; i++) { + ps.setString(1, updateString); + ps.setInt(2, key); + ps.executeUpdate(); + } + } + + public static void main(String[] argv) throws SQLException, IllegalAccessException, ClassNotFoundException, InstantiationException, InterruptedException, IOException { + ij.getPropertyArg(argv); + Connection conn = ij.startJBMS(); + Statement s = conn.createStatement(); + s + .executeUpdate("CREATE TABLE CLOBTAB (I INT PRIMARY KEY NOT NULL, c CLOB)"); + PreparedStatement ps = conn + .prepareStatement("INSERT INTO CLOBTAB VALUES(?,?)"); + String insertString = "hello"; + for (int i = 1; i <= NUM_THREADS; i++) { + ps.setInt(1, i); + ps.setString(2, insertString); + ps.executeUpdate(); + } + testMultiThreadedUpdate(conn); + + } +} \ No newline at end of file Propchange: db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/tests/store/ClobReclamationTest.java ------------------------------------------------------------------------------ svn:eol-style = native Added: db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/tests/store/ClobReclamationTest_derby.properties URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/tests/store/ClobReclamationTest_derby.properties?rev=743798&view=auto ============================================================================== --- db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/tests/store/ClobReclamationTest_derby.properties (added) +++ db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/tests/store/ClobReclamationTest_derby.properties Thu Feb 12 16:40:16 2009 @@ -0,0 +1 @@ +derby.debug.true=DaemonTrace \ No newline at end of file Propchange: db/derby/code/branches/10.2/java/testing/org/apache/derbyTesting/functionTests/tests/store/ClobReclamationTest_derby.properties ------------------------------------------------------------------------------ svn:eol-style = native