Mailing-List: contact derby-commits-help@db.apache.org; run by ezmlm
Precedence: bulk
Reply-To: "Derby Development" <derby-dev@db.apache.org>
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1304848 - in /db/derby/code/branches/10.7/java:
 engine/org/apache/derby/impl/sql/compile/
 engine/org/apache/derby/impl/sql/execute/
 testing/org/apache/derbyTesting/functionTests/tests/lang/
Date: Sat, 24 Mar 2012 16:27:41 -0000
To: derby-commits@db.apache.org
From: mikem@apache.org
Message-Id: <20120324162741.C0294238896F@eris.apache.org>

Author: mikem
Date: Sat Mar 24 16:27:41 2012
New Revision: 1304848

URL: http://svn.apache.org/viewvc?rev=1304848&view=rev
Log:
DERBY-4275: Query executions fail when compressing a table using SYSCS_UTIL.SYSCS_COMPRESS_TABLE

backported changes #1142583 and #1160597 from trunk to 10.7 branch.

Move invalidation of dependent statements until the system tables have
been updated with information about the new conglomerates created by
compression or truncation. This is to prevent that statements
executing concurrently get recompiled too early and don't see the new
conglomerates (and therefore fail on subsequent executions because
they cannot find the old conglomerates). 

Fail in a controlled fashion (StandardException) if the conglomerate
disappears while binding the FromBaseTable. This used to cause a
NullPointerException. 


Modified:
    db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/sql/compile/FromBaseTable.java
    db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/sql/execute/AlterTableConstantAction.java
    db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/CompressTableTest.java
    db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/TruncateTableTest.java

Modified: db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/sql/compile/FromBaseTable.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/sql/compile/FromBaseTable.java?rev=1304848&r1=1304847&r2=1304848&view=diff
==============================================================================
--- db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/sql/compile/FromBaseTable.java (original)
+++ db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/sql/compile/FromBaseTable.java Sat Mar 24 16:27:41 2012
@@ -2324,6 +2324,14 @@ public class FromBaseTable extends FromT
 					tableDescriptor.getHeapConglomerateId()
 					);
 
+            // Bail out if the descriptor couldn't be found. The conglomerate
+            // probably doesn't exist anymore.
+            if (baseConglomerateDescriptor == null) {
+                throw StandardException.newException(
+                        SQLState.STORE_CONGLOMERATE_DOES_NOT_EXIST,
+                        new Long(tableDescriptor.getHeapConglomerateId()));
+            }
+
 			/* Build the 0-based array of base column names. */
 			columnNames = resultColumns.getColumnNames();
 

Modified: db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/sql/execute/AlterTableConstantAction.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/sql/execute/AlterTableConstantAction.java?rev=1304848&r1=1304847&r2=1304848&view=diff
==============================================================================
--- db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/sql/execute/AlterTableConstantAction.java (original)
+++ db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/sql/execute/AlterTableConstantAction.java Sat Mar 24 16:27:41 2012
@@ -398,7 +398,7 @@ class AlterTableConstantAction extends D
 			sd = getAndCheckSchemaDescriptor(dd, schemaId, "ALTER TABLE");
 		}
 		
-		/* Prepare all dependents to invalidate.  (This is there chance
+		/* Prepare all dependents to invalidate.  (This is their chance
 		 * to say that they can't be invalidated.  For example, an open
 		 * cursor referencing a table/view that the user is attempting to
 		 * alter.) If no one objects, then invalidate any dependent objects.
@@ -2290,14 +2290,6 @@ class AlterTableConstantAction extends D
                 TransactionController.MODE_TABLE,
                 TransactionController.ISOLATION_SERIALIZABLE);
 
-		// invalidate any prepared statements that depended on this table 
-        // (including this one), this fixes problem with threads that start up 
-        // and block on our lock, but do not see they have to recompile their 
-        // plan.  We now invalidate earlier however they still might recompile
-        // using the old conglomerate id before we commit our DD changes.
-		//
-		dm.invalidateFor(td, DependencyManager.COMPRESS_TABLE, lcc);
-
 		rl = compressHeapCC.newRowLocationTemplate();
 
 		// Get the properties on the old heap
@@ -2421,6 +2413,10 @@ class AlterTableConstantAction extends D
 		// Update sys.sysconglomerates with new conglomerate #
 		dd.updateConglomerateDescriptor(cd, newHeapConglom, tc);
 
+        // Now that the updated information is available in the system tables,
+        // we should invalidate all statements that use the old conglomerates
+        dm.invalidateFor(td, DependencyManager.COMPRESS_TABLE, lcc);
+
 		// Drop the old conglomerate
 		tc.dropConglomerate(oldHeapConglom);
 		cleanUp();
@@ -2505,15 +2501,6 @@ class AlterTableConstantAction extends D
                                 TransactionController.MODE_TABLE,
                                 TransactionController.ISOLATION_SERIALIZABLE);
 
-		// invalidate any prepared statements that
-		// depended on this table (including this one)
-		// bug 3653 has threads that start up and block on our lock, but do
-		// not see they have to recompile their plan.    We now invalidate earlier
-		// however they still might recompile using the old conglomerate id before we
-		// commit our DD changes.
-		//
-		dm.invalidateFor(td, DependencyManager.TRUNCATE_TABLE, lcc);
-
 		rl = compressHeapCC.newRowLocationTemplate();
 		// Get the properties on the old heap
 		compressHeapCC.getInternalTablePropertySet(properties);
@@ -2600,6 +2587,11 @@ class AlterTableConstantAction extends D
 
 		// Update sys.sysconglomerates with new conglomerate #
 		dd.updateConglomerateDescriptor(cd, newHeapConglom, tc);
+
+        // Now that the updated information is available in the system tables,
+        // we should invalidate all statements that use the old conglomerates
+        dm.invalidateFor(td, DependencyManager.TRUNCATE_TABLE, lcc);
+
 		// Drop the old conglomerate
 		tc.dropConglomerate(oldHeapConglom);
 		cleanUp();

Modified: db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/CompressTableTest.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/CompressTableTest.java?rev=1304848&r1=1304847&r2=1304848&view=diff
==============================================================================
--- db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/CompressTableTest.java (original)
+++ db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/CompressTableTest.java Sat Mar 24 16:27:41 2012
@@ -21,10 +21,17 @@ limitations under the License.
 
 package org.apache.derbyTesting.functionTests.tests.lang;
 
+import java.sql.Connection;
+import java.sql.PreparedStatement;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import junit.framework.Test;
 import org.apache.derbyTesting.junit.BaseJDBCTestCase;
+import org.apache.derbyTesting.junit.CleanDatabaseTestSetup;
+import org.apache.derbyTesting.junit.JDBC;
 import org.apache.derbyTesting.junit.TestConfiguration;
 
 /**
@@ -38,7 +45,8 @@ public class CompressTableTest extends B
 
     public static Test suite() {
         // compress table is an embedded feature, no need to run network tests
-        return TestConfiguration.embeddedSuite(CompressTableTest.class);
+        return new CleanDatabaseTestSetup(
+                TestConfiguration.embeddedSuite(CompressTableTest.class));
     }
 
     /**
@@ -54,4 +62,67 @@ public class CompressTableTest extends B
                   "'abc\"def', 1, 1, 1)");
         s.execute("drop table app.\"abc\"\"def\"");
     }
+
+    /**
+     * Test that statement invalidation works when SYSCS_COMPRESS_TABLE calls
+     * and other statements accessing the same table execute concurrently.
+     * DERBY-4275.
+     */
+    public void testConcurrentInvalidation() throws Exception {
+        Statement s = createStatement();
+        s.execute("create table d4275(x int)");
+        s.execute("insert into d4275 values 1");
+
+        // Object used by the main thread to tell the helper thread to stop.
+        // The helper thread stops once the list is non-empty.
+        final List stop = Collections.synchronizedList(new ArrayList());
+
+        // Holder for anything thrown by the run() method in the helper thread.
+        final Throwable[] error = new Throwable[1];
+
+        // Set up a helper thread that executes a query against the table
+        // until the main thread tells it to stop.
+        Connection c2 = openDefaultConnection();
+        final PreparedStatement ps = c2.prepareStatement("select * from d4275");
+
+        Thread t = new Thread() {
+            public void run() {
+                try {
+                    while (stop.isEmpty()) {
+                        JDBC.assertSingleValueResultSet(ps.executeQuery(), "1");
+                    }
+                } catch (Throwable t) {
+                    error[0] = t;
+                }
+            }
+        };
+
+        t.start();
+
+        // Compress the table while a query is being executed against the
+        // same table to force invalidation of the running statement. Since
+        // the problem we try to reproduce is timing-dependent, do it 100
+        // times to increase the chance of hitting the bug.
+        try {
+            for (int i = 0; i < 100; i++) {
+                s.execute(
+                    "call syscs_util.syscs_compress_table('APP', 'D4275', 1)");
+            }
+        } finally {
+            // We're done, so tell the helper thread to stop.
+            stop.add(Boolean.TRUE);
+        }
+
+        t.join();
+
+        // Before DERBY-4275, the helper thread used to fail with an error
+        // saying the container was not found.
+        if (error[0] != null) {
+            fail("Helper thread failed", error[0]);
+        }
+
+        // Cleanup.
+        ps.close();
+        c2.close();
+    }
 }

Modified: db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/TruncateTableTest.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/TruncateTableTest.java?rev=1304848&r1=1304847&r2=1304848&view=diff
==============================================================================
--- db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/TruncateTableTest.java (original)
+++ db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/TruncateTableTest.java Sat Mar 24 16:27:41 2012
@@ -26,6 +26,9 @@ import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import junit.framework.Test;
 import org.apache.derbyTesting.junit.BaseJDBCTestCase;
 import org.apache.derbyTesting.junit.DatabasePropertyTestSetup;
@@ -286,4 +289,64 @@ public class TruncateTableTest extends B
         truncatorStatement.close();
     }
     
+    /**
+     * Test that statement invalidation works when TRUNCATE TABLE statements
+     * and other statements accessing the same table execute concurrently.
+     * DERBY-4275.
+     */
+    public void testConcurrentInvalidation() throws Exception {
+        Statement s = createStatement();
+        s.execute("create table d4275(x int)");
+
+        // Object used by the main thread to tell the helper thread to stop.
+        // The helper thread stops once the list is non-empty.
+        final List stop = Collections.synchronizedList(new ArrayList());
+
+        // Holder for anything thrown by the run() method in the helper thread.
+        final Throwable[] error = new Throwable[1];
+
+        // Set up a helper thread that executes a query against the table
+        // until the main thread tells it to stop.
+        Connection c2 = openDefaultConnection();
+        final PreparedStatement ps = c2.prepareStatement("select * from d4275");
+
+        Thread t = new Thread() {
+            public void run() {
+                try {
+                    while (stop.isEmpty()) {
+                        JDBC.assertEmpty(ps.executeQuery());
+                    }
+                } catch (Throwable t) {
+                    error[0] = t;
+                }
+            }
+        };
+
+        t.start();
+
+        // Truncate the table while a query is being executed against the
+        // same table to force invalidation of the running statement. Since
+        // the problem we try to reproduce is timing-dependent, do it 100
+        // times to increase the chance of hitting the bug.
+        try {
+            for (int i = 0; i < 100; i++) {
+                s.execute("truncate table d4275");
+            }
+        } finally {
+            // We're done, so tell the helper thread to stop.
+            stop.add(Boolean.TRUE);
+        }
+
+        t.join();
+
+        // Before DERBY-4275, the helper thread used to fail with an error
+        // saying the container was not found.
+        if (error[0] != null) {
+            fail("Helper thread failed", error[0]);
+        }
+
+        // Cleanup.
+        ps.close();
+        c2.close();
+    }
 }