Return-Path: Delivered-To: apmail-db-derby-commits-archive@www.apache.org Received: (qmail 91040 invoked from network); 14 Apr 2011 09:08:33 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 14 Apr 2011 09:08:33 -0000 Received: (qmail 96463 invoked by uid 500); 14 Apr 2011 09:08:33 -0000 Delivered-To: apmail-db-derby-commits-archive@db.apache.org Received: (qmail 96406 invoked by uid 500); 14 Apr 2011 09:08:32 -0000 Mailing-List: contact derby-commits-help@db.apache.org; run by ezmlm Precedence: bulk list-help: list-unsubscribe: List-Post: Reply-To: "Derby Development" List-Id: Delivered-To: mailing list derby-commits@db.apache.org Received: (qmail 96395 invoked by uid 99); 14 Apr 2011 09:08:31 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 14 Apr 2011 09:08:31 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 14 Apr 2011 09:08:29 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 47F0623888FE; Thu, 14 Apr 2011 09:08:09 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1092067 - in /db/derby/code/trunk/java: engine/org/apache/derby/impl/store/access/btree/index/ engine/org/apache/derby/impl/store/access/heap/ engine/org/apache/derby/impl/store/raw/data/ testing/org/apache/derbyTesting/functionTests/tests... Date: Thu, 14 Apr 2011 09:08:09 -0000 To: derby-commits@db.apache.org From: kahatlen@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110414090809.47F0623888FE@eris.apache.org> Author: kahatlen Date: Thu Apr 14 09:08:08 2011 New Revision: 1092067 URL: http://svn.apache.org/viewvc?rev=1092067&view=rev Log: DERBY-2354: Unable to perform select query using DISTINCT on a read-only database Made databases that are stored in a jar or a zip use a writable directory for temporary files needed by some operations (typically hash scans when the hash table gets too big to be kept in memory). The writable directory can be controlled by the database property derby.storage.tempDirectory. Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java?rev=1092067&r1=1092066&r2=1092067&view=diff ============================================================================== --- db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java (original) +++ db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java Thu Apr 14 09:08:08 2011 @@ -36,6 +36,7 @@ import org.apache.derby.iapi.store.acces import org.apache.derby.iapi.store.access.conglomerate.ConglomerateFactory; import org.apache.derby.iapi.store.access.conglomerate.TransactionManager; import org.apache.derby.iapi.store.access.ColumnOrdering; +import org.apache.derby.iapi.store.access.TransactionController; import org.apache.derby.iapi.store.raw.ContainerKey; import org.apache.derby.iapi.store.raw.ContainerHandle; @@ -173,7 +174,17 @@ public class B2IFactory implements Congl { B2I btree = null; - if (xact_mgr.checkVersion( + if ((temporaryFlag & TransactionController.IS_TEMPORARY) != 0 && + xact_mgr.getAccessManager().isReadOnly()) + { + // If this is a temporary conglomerate created for a read-only + // database, we don't really care which disk format we use, since + // it is not used for persisting data in the database. Use the + // current format. A special case is needed because checkVersion() + // throws an exception in read-only databases (DERBY-2354). + btree = new B2I(); + } + else if (xact_mgr.checkVersion( RawStoreFactory.DERBY_STORE_MAJOR_VERSION_10, RawStoreFactory.DERBY_STORE_MINOR_VERSION_4, null)) Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java?rev=1092067&r1=1092066&r2=1092067&view=diff ============================================================================== --- db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java (original) +++ db/derby/code/trunk/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java Thu Apr 14 09:08:08 2011 @@ -34,6 +34,7 @@ import org.apache.derby.iapi.store.acces import org.apache.derby.iapi.store.access.conglomerate.TransactionManager; import org.apache.derby.iapi.store.access.ColumnOrdering; +import org.apache.derby.iapi.store.access.TransactionController; import org.apache.derby.iapi.store.raw.ContainerHandle; import org.apache.derby.iapi.store.raw.FetchDescriptor; @@ -172,7 +173,17 @@ public class HeapConglomerateFactory imp Heap heap = null; - if (xact_mgr.checkVersion( + if ((temporaryFlag & TransactionController.IS_TEMPORARY) != 0 && + xact_mgr.getAccessManager().isReadOnly()) + { + // If this is a temporary conglomerate created for a read-only + // database, we don't really care which disk format we use, since + // it is not used for persisting data in the database. Use the + // current format. A special case is needed because checkVersion() + // throws an exception in read-only databases (DERBY-2354). + heap = new Heap(); + } + else if (xact_mgr.checkVersion( RawStoreFactory.DERBY_STORE_MAJOR_VERSION_10, RawStoreFactory.DERBY_STORE_MINOR_VERSION_3, null)) Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java?rev=1092067&r1=1092066&r2=1092067&view=diff ============================================================================== --- db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java (original) +++ db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java Thu Apr 14 09:08:08 2011 @@ -347,7 +347,15 @@ abstract class FileContainer */ public Cacheable setIdentity(Object key) throws StandardException { - return setIdent((ContainerKey) key); + ContainerKey newIdentity = (ContainerKey) key; + + // If the new identity represents a temporary container, switch to + // TempRAFContainer. + if (newIdentity.getSegmentId() == ContainerHandle.TEMPORARY_SEGMENT) { + return new TempRAFContainer(dataFactory).setIdent(newIdentity); + } + + return setIdent(newIdentity); } /** @@ -389,13 +397,16 @@ abstract class FileContainer public Cacheable createIdentity(Object key, Object createParameter) throws StandardException { - if (SanityManager.DEBUG) - { - SanityManager.ASSERT( - !(key instanceof PageKey), "PageKey input to create container"); - } + ContainerKey newIdentity = (ContainerKey) key; + + // If the new identity represents a temporary container, switch to + // TempRAFContainer. + if (newIdentity.getSegmentId() == ContainerHandle.TEMPORARY_SEGMENT) { + TempRAFContainer tmpContainer = new TempRAFContainer(dataFactory); + return tmpContainer.createIdent(newIdentity, createParameter); + } - return createIdent((ContainerKey) key, createParameter); + return createIdent(newIdentity, createParameter); } Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java?rev=1092067&r1=1092066&r2=1092067&view=diff ============================================================================== --- db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java (original) +++ db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java Thu Apr 14 09:08:08 2011 @@ -22,14 +22,7 @@ package org.apache.derby.impl.store.raw.data; import org.apache.derby.iapi.reference.SQLState; -import org.apache.derby.impl.store.raw.data.BaseContainer; -import org.apache.derby.impl.store.raw.data.BaseContainerHandle; -import org.apache.derby.impl.store.raw.data.BasePage; - -import org.apache.derby.iapi.services.cache.Cacheable; -import org.apache.derby.iapi.services.context.ContextService; -import org.apache.derby.iapi.services.monitor.Monitor; -import org.apache.derby.iapi.services.diag.Performance; + import org.apache.derby.iapi.services.sanity.SanityManager; import org.apache.derby.iapi.services.io.FormatIdUtil; @@ -40,18 +33,12 @@ import org.apache.derby.iapi.error.Stand import org.apache.derby.iapi.store.raw.ContainerHandle; import org.apache.derby.iapi.store.raw.ContainerKey; -import org.apache.derby.iapi.store.raw.Loggable; import org.apache.derby.iapi.store.raw.log.LogInstant; -import org.apache.derby.iapi.store.raw.xact.RawTransaction; -import org.apache.derby.io.StorageFactory; -import org.apache.derby.io.WritableStorageFactory; import org.apache.derby.io.StorageFile; import org.apache.derby.io.StorageRandomAccessFile; import org.apache.derby.iapi.services.io.FileUtil; -import java.util.Vector; -import java.io.DataInput; import java.io.IOException; import java.io.File; import java.io.FileNotFoundException; @@ -127,44 +114,6 @@ class RAFContainer extends FileContainer } /* - ** Methods of Cacheable - */ - - /** - Set container's identity - @exception StandardException Standard Derby error policy - */ - public Cacheable setIdentity(Object key) throws StandardException { - - ContainerKey newIdentity = (ContainerKey) key; - - // if this is an open for a temp container then return an object of that type - if (newIdentity.getSegmentId() == ContainerHandle.TEMPORARY_SEGMENT) { - - TempRAFContainer tmpContainer = new TempRAFContainer(dataFactory); - return tmpContainer.setIdent(newIdentity); - } - - return setIdent(newIdentity); - } - - /** - @exception StandardException Standard Derby error policy - */ - public Cacheable createIdentity(Object key, Object createParameter) throws StandardException { - - ContainerKey newIdentity = (ContainerKey) key; - - if (newIdentity.getSegmentId() == ContainerHandle.TEMPORARY_SEGMENT) { - TempRAFContainer tmpContainer = new TempRAFContainer(dataFactory); - return tmpContainer.createIdent(newIdentity, createParameter); - } - - return createIdent(newIdentity, createParameter); - } - - - /* ** Container creation, opening, and closing */ Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java?rev=1092067&r1=1092066&r2=1092067&view=diff ============================================================================== --- db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java (original) +++ db/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java Thu Apr 14 09:08:08 2011 @@ -65,7 +65,8 @@ class TempRAFContainer extends RAFContai ContainerKey newIdentity = (ContainerKey) key; if (newIdentity.getSegmentId() != ContainerHandle.TEMPORARY_SEGMENT) { - RAFContainer realContainer = new RAFContainer(dataFactory); + FileContainer realContainer = + (FileContainer) dataFactory.newContainerObject(); return realContainer.setIdent(newIdentity); } @@ -81,7 +82,7 @@ class TempRAFContainer extends RAFContai ContainerKey newIdentity = (ContainerKey) key; if (newIdentity.getSegmentId() != ContainerHandle.TEMPORARY_SEGMENT) { - RAFContainer realContainer = new RAFContainer(dataFactory); + Cacheable realContainer = dataFactory.newContainerObject(); return realContainer.createIdentity(newIdentity, createParameter); } Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java?rev=1092067&r1=1092066&r2=1092067&view=diff ============================================================================== --- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java (original) +++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java Thu Apr 14 09:08:08 2011 @@ -28,18 +28,15 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.sql.DriverManager; - +import java.sql.PreparedStatement; import junit.framework.Test; import junit.framework.TestSuite; -import org.apache.derbyTesting.functionTests.tests.jdbcapi.BatchUpdateTest; import org.apache.derbyTesting.junit.BaseJDBCTestCase; import org.apache.derbyTesting.junit.CleanDatabaseTestSetup; import org.apache.derbyTesting.junit.JDBC; import org.apache.derbyTesting.junit.SecurityManagerSetup; -import org.apache.derbyTesting.junit.TestConfiguration; -import org.apache.derbyTesting.junit.Utilities; public class DBInJarTest extends BaseJDBCTestCase { @@ -105,6 +102,97 @@ public class DBInJarTest extends BaseJDB } } + /** + * Test various queries that use a hash table that may be spilled to disk + * if it grows too big. Regression test case for DERBY-2354. + */ + public void testSpillHashToDisk() throws SQLException { + createDerby2354Database(); + + Connection jarConn = + DriverManager.getConnection("jdbc:derby:jar:(d2354db.jar)d2354db"); + + Statement stmt = jarConn.createStatement(); + + // The following statement used to fail with "Feature not implemented" + // or "Container was opened in read-only mode" before DERBY-2354. It + // only fails if the hash table used for duplicate elimination spills + // to disk, which happens if the hash table gets bigger than 1% of the + // total amount of memory allocated to the JVM. This means it won't + // expose the bug if the JVM runs with very high memory settings (but + // it has been tested with 1 GB heap size and then it did spill to + // disk). + JDBC.assertDrainResults( + stmt.executeQuery("select distinct x from d2354"), + 40000); + + // Hash joins have the same problem. Force the big table to be used as + // the inner table in the hash join. + JDBC.assertEmpty(stmt.executeQuery( + "select * from --DERBY-PROPERTIES joinOrder = FIXED\n" + + "sysibm.sysdummy1 t1(x),\n" + + "d2354 t2 --DERBY-PROPERTIES joinStrategy = HASH\n" + + "where t1.x = t2.x")); + + // Scrollable result sets keep the rows they've visited in a hash + // table, so they may also need to store data on disk temporarily. + Statement scrollStmt = jarConn.createStatement( + ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY); + JDBC.assertDrainResults( + scrollStmt.executeQuery("select * from d2354"), + 40000); + + stmt.close(); + scrollStmt.close(); + jarConn.close(); + + // Cleanup. Shut down the database and delete it. + shutdownDB("jdbc:derby:jar:(d2354db.jar)d2354db;shutdown=true"); + removeFiles(new String[] { + System.getProperty("derby.system.home") + "/d2354db.jar" + }); + } + + /** + * Create a database in a jar for use in {@code testSpillHashToDisk}. + */ + private void createDerby2354Database() throws SQLException { + // First create an ordinary database with a table. + Connection conn = + DriverManager.getConnection("jdbc:derby:d2354db;create=true"); + conn.setAutoCommit(false); + Statement s = conn.createStatement(); + s.execute("create table d2354 (x varchar(100))"); + s.close(); + + // Insert 40000 unique values into the table. The values should be + // unique so that they all occupy an entry in the hash table used by + // the DISTINCT query in the test, and thereby increase the likelihood + // of spilling to disk. + PreparedStatement insert = + conn.prepareStatement( + "insert into d2354 values ? || " + + "'some extra data to increase the size of the table'"); + for (int i = 0; i < 40000; i++) { + insert.setInt(1, i); + insert.executeUpdate(); + } + insert.close(); + + conn.commit(); + conn.close(); + + // Shut down the database and archive it in a jar file. + shutdownDB("jdbc:derby:d2354db;shutdown=true"); + + createStatement().execute( + "CALL CREATEARCHIVE('d2354db.jar', 'd2354db', 'd2354db')"); + + // Clean up the original database directory. We don't need it anymore + // now that we have archived it in a jar file. + removeDirectory( + new File(System.getProperty("derby.system.home") + "/d2354db")); + } protected static Test baseSuite(String name) { TestSuite suite = new TestSuite(name);