From derby-commits-return-16045-apmail-db-derby-commits-archive=db.apache.org@db.apache.org Tue Sep 25 01:36:49 2012 Return-Path: X-Original-To: apmail-db-derby-commits-archive@www.apache.org Delivered-To: apmail-db-derby-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 51F53D530 for ; Tue, 25 Sep 2012 01:36:49 +0000 (UTC) Received: (qmail 92526 invoked by uid 500); 25 Sep 2012 01:36:48 -0000 Delivered-To: apmail-db-derby-commits-archive@db.apache.org Received: (qmail 92401 invoked by uid 500); 25 Sep 2012 01:36:47 -0000 Mailing-List: contact derby-commits-help@db.apache.org; run by ezmlm Precedence: bulk list-help: list-unsubscribe: List-Post: Reply-To: "Derby Development" List-Id: Delivered-To: mailing list derby-commits@db.apache.org Received: (qmail 92393 invoked by uid 99); 25 Sep 2012 01:36:47 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 25 Sep 2012 01:36:47 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 25 Sep 2012 01:36:43 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 10C8723888EA; Tue, 25 Sep 2012 01:35:59 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1389677 - in /db/derby/code/branches/10.7: ./ java/engine/org/apache/derby/impl/store/access/btree/index/ java/engine/org/apache/derby/impl/store/access/heap/ java/engine/org/apache/derby/impl/store/raw/data/ java/testing/org/apache/derbyT... Date: Tue, 25 Sep 2012 01:35:58 -0000 To: derby-commits@db.apache.org From: mamta@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120925013559.10C8723888EA@eris.apache.org> Author: mamta Date: Tue Sep 25 01:35:57 2012 New Revision: 1389677 URL: http://svn.apache.org/viewvc?rev=1389677&view=rev Log: DERBY-2354 (Unable to perform select query using DISTINCT on a read-only database) Backporting this jira from 10.8 commit. Additionally, I hand backported changes to BaseTestCase and DropDatabaseSetup from another checkin so DBInJarTest has access to supporting method from BaseTestCase Modified: db/derby/code/branches/10.7/ (props changed) db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/BaseTestCase.java db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/DropDatabaseSetup.java Propchange: db/derby/code/branches/10.7/ ------------------------------------------------------------------------------ Merged /db/derby/code/trunk:r1092067 Merged /db/derby/code/branches/10.8:r1092649 Modified: db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java?rev=1389677&r1=1389676&r2=1389677&view=diff ============================================================================== --- db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java (original) +++ db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/btree/index/B2IFactory.java Tue Sep 25 01:35:57 2012 @@ -36,6 +36,7 @@ import org.apache.derby.iapi.store.acces import org.apache.derby.iapi.store.access.conglomerate.ConglomerateFactory; import org.apache.derby.iapi.store.access.conglomerate.TransactionManager; import org.apache.derby.iapi.store.access.ColumnOrdering; +import org.apache.derby.iapi.store.access.TransactionController; import org.apache.derby.iapi.store.raw.ContainerKey; import org.apache.derby.iapi.store.raw.ContainerHandle; @@ -173,7 +174,17 @@ public class B2IFactory implements Congl { B2I btree = null; - if (xact_mgr.checkVersion( + if ((temporaryFlag & TransactionController.IS_TEMPORARY) != 0 && + xact_mgr.getAccessManager().isReadOnly()) + { + // If this is a temporary conglomerate created for a read-only + // database, we don't really care which disk format we use, since + // it is not used for persisting data in the database. Use the + // current format. A special case is needed because checkVersion() + // throws an exception in read-only databases (DERBY-2354). + btree = new B2I(); + } + else if (xact_mgr.checkVersion( RawStoreFactory.DERBY_STORE_MAJOR_VERSION_10, RawStoreFactory.DERBY_STORE_MINOR_VERSION_4, null)) Modified: db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java?rev=1389677&r1=1389676&r2=1389677&view=diff ============================================================================== --- db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java (original) +++ db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/access/heap/HeapConglomerateFactory.java Tue Sep 25 01:35:57 2012 @@ -34,6 +34,7 @@ import org.apache.derby.iapi.store.acces import org.apache.derby.iapi.store.access.conglomerate.TransactionManager; import org.apache.derby.iapi.store.access.ColumnOrdering; +import org.apache.derby.iapi.store.access.TransactionController; import org.apache.derby.iapi.store.raw.ContainerHandle; import org.apache.derby.iapi.store.raw.FetchDescriptor; @@ -172,7 +173,17 @@ public class HeapConglomerateFactory imp Heap heap = null; - if (xact_mgr.checkVersion( + if ((temporaryFlag & TransactionController.IS_TEMPORARY) != 0 && + xact_mgr.getAccessManager().isReadOnly()) + { + // If this is a temporary conglomerate created for a read-only + // database, we don't really care which disk format we use, since + // it is not used for persisting data in the database. Use the + // current format. A special case is needed because checkVersion() + // throws an exception in read-only databases (DERBY-2354). + heap = new Heap(); + } + else if (xact_mgr.checkVersion( RawStoreFactory.DERBY_STORE_MAJOR_VERSION_10, RawStoreFactory.DERBY_STORE_MINOR_VERSION_3, null)) Modified: db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java?rev=1389677&r1=1389676&r2=1389677&view=diff ============================================================================== --- db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java (original) +++ db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java Tue Sep 25 01:35:57 2012 @@ -344,7 +344,15 @@ abstract class FileContainer */ public Cacheable setIdentity(Object key) throws StandardException { - return setIdent((ContainerKey) key); + ContainerKey newIdentity = (ContainerKey) key; + + // If the new identity represents a temporary container, switch to + // TempRAFContainer. + if (newIdentity.getSegmentId() == ContainerHandle.TEMPORARY_SEGMENT) { + return new TempRAFContainer(dataFactory).setIdent(newIdentity); + } + + return setIdent(newIdentity); } /** @@ -386,13 +394,16 @@ abstract class FileContainer public Cacheable createIdentity(Object key, Object createParameter) throws StandardException { - if (SanityManager.DEBUG) - { - SanityManager.ASSERT( - !(key instanceof PageKey), "PageKey input to create container"); - } + ContainerKey newIdentity = (ContainerKey) key; + + // If the new identity represents a temporary container, switch to + // TempRAFContainer. + if (newIdentity.getSegmentId() == ContainerHandle.TEMPORARY_SEGMENT) { + TempRAFContainer tmpContainer = new TempRAFContainer(dataFactory); + return tmpContainer.createIdent(newIdentity, createParameter); + } - return createIdent((ContainerKey) key, createParameter); + return createIdent(newIdentity, createParameter); } Modified: db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java?rev=1389677&r1=1389676&r2=1389677&view=diff ============================================================================== --- db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java (original) +++ db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java Tue Sep 25 01:35:57 2012 @@ -22,14 +22,7 @@ package org.apache.derby.impl.store.raw.data; import org.apache.derby.iapi.reference.SQLState; -import org.apache.derby.impl.store.raw.data.BaseContainer; -import org.apache.derby.impl.store.raw.data.BaseContainerHandle; -import org.apache.derby.impl.store.raw.data.BasePage; - -import org.apache.derby.iapi.services.cache.Cacheable; -import org.apache.derby.iapi.services.context.ContextService; -import org.apache.derby.iapi.services.monitor.Monitor; -import org.apache.derby.iapi.services.diag.Performance; + import org.apache.derby.iapi.services.sanity.SanityManager; import org.apache.derby.iapi.services.io.FormatIdUtil; @@ -37,18 +30,12 @@ import org.apache.derby.iapi.error.Stand import org.apache.derby.iapi.store.raw.ContainerHandle; import org.apache.derby.iapi.store.raw.ContainerKey; -import org.apache.derby.iapi.store.raw.Loggable; import org.apache.derby.iapi.store.raw.log.LogInstant; -import org.apache.derby.iapi.store.raw.xact.RawTransaction; -import org.apache.derby.io.StorageFactory; -import org.apache.derby.io.WritableStorageFactory; import org.apache.derby.io.StorageFile; import org.apache.derby.io.StorageRandomAccessFile; import org.apache.derby.iapi.services.io.FileUtil; -import java.util.Vector; -import java.io.DataInput; import java.io.IOException; import java.io.File; import java.io.RandomAccessFile; @@ -114,44 +101,6 @@ class RAFContainer extends FileContainer } /* - ** Methods of Cacheable - */ - - /** - Set container's identity - @exception StandardException Standard Derby error policy - */ - public Cacheable setIdentity(Object key) throws StandardException { - - ContainerKey newIdentity = (ContainerKey) key; - - // if this is an open for a temp container then return an object of that type - if (newIdentity.getSegmentId() == ContainerHandle.TEMPORARY_SEGMENT) { - - TempRAFContainer tmpContainer = new TempRAFContainer(dataFactory); - return tmpContainer.setIdent(newIdentity); - } - - return setIdent(newIdentity); - } - - /** - @exception StandardException Standard Derby error policy - */ - public Cacheable createIdentity(Object key, Object createParameter) throws StandardException { - - ContainerKey newIdentity = (ContainerKey) key; - - if (newIdentity.getSegmentId() == ContainerHandle.TEMPORARY_SEGMENT) { - TempRAFContainer tmpContainer = new TempRAFContainer(dataFactory); - return tmpContainer.createIdent(newIdentity, createParameter); - } - - return createIdent(newIdentity, createParameter); - } - - - /* ** Container creation, opening, and closing */ Modified: db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java?rev=1389677&r1=1389676&r2=1389677&view=diff ============================================================================== --- db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java (original) +++ db/derby/code/branches/10.7/java/engine/org/apache/derby/impl/store/raw/data/TempRAFContainer.java Tue Sep 25 01:35:57 2012 @@ -65,7 +65,8 @@ class TempRAFContainer extends RAFContai ContainerKey newIdentity = (ContainerKey) key; if (newIdentity.getSegmentId() != ContainerHandle.TEMPORARY_SEGMENT) { - RAFContainer realContainer = new RAFContainer(dataFactory); + FileContainer realContainer = + (FileContainer) dataFactory.newContainerObject(); return realContainer.setIdent(newIdentity); } @@ -81,7 +82,7 @@ class TempRAFContainer extends RAFContai ContainerKey newIdentity = (ContainerKey) key; if (newIdentity.getSegmentId() != ContainerHandle.TEMPORARY_SEGMENT) { - RAFContainer realContainer = new RAFContainer(dataFactory); + Cacheable realContainer = dataFactory.newContainerObject(); return realContainer.createIdentity(newIdentity, createParameter); } Modified: db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java?rev=1389677&r1=1389676&r2=1389677&view=diff ============================================================================== --- db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java (original) +++ db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/functionTests/tests/lang/DBInJarTest.java Tue Sep 25 01:35:57 2012 @@ -28,18 +28,15 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.sql.DriverManager; - +import java.sql.PreparedStatement; import junit.framework.Test; import junit.framework.TestSuite; -import org.apache.derbyTesting.functionTests.tests.jdbcapi.BatchUpdateTest; import org.apache.derbyTesting.junit.BaseJDBCTestCase; import org.apache.derbyTesting.junit.CleanDatabaseTestSetup; import org.apache.derbyTesting.junit.JDBC; import org.apache.derbyTesting.junit.SecurityManagerSetup; -import org.apache.derbyTesting.junit.TestConfiguration; -import org.apache.derbyTesting.junit.Utilities; public class DBInJarTest extends BaseJDBCTestCase { @@ -105,6 +102,97 @@ public class DBInJarTest extends BaseJDB } } + /** + * Test various queries that use a hash table that may be spilled to disk + * if it grows too big. Regression test case for DERBY-2354. + */ + public void testSpillHashToDisk() throws SQLException { + createDerby2354Database(); + + Connection jarConn = + DriverManager.getConnection("jdbc:derby:jar:(d2354db.jar)d2354db"); + + Statement stmt = jarConn.createStatement(); + + // The following statement used to fail with "Feature not implemented" + // or "Container was opened in read-only mode" before DERBY-2354. It + // only fails if the hash table used for duplicate elimination spills + // to disk, which happens if the hash table gets bigger than 1% of the + // total amount of memory allocated to the JVM. This means it won't + // expose the bug if the JVM runs with very high memory settings (but + // it has been tested with 1 GB heap size and then it did spill to + // disk). + JDBC.assertDrainResults( + stmt.executeQuery("select distinct x from d2354"), + 40000); + + // Hash joins have the same problem. Force the big table to be used as + // the inner table in the hash join. + JDBC.assertEmpty(stmt.executeQuery( + "select * from --DERBY-PROPERTIES joinOrder = FIXED\n" + + "sysibm.sysdummy1 t1(x),\n" + + "d2354 t2 --DERBY-PROPERTIES joinStrategy = HASH\n" + + "where t1.x = t2.x")); + + // Scrollable result sets keep the rows they've visited in a hash + // table, so they may also need to store data on disk temporarily. + Statement scrollStmt = jarConn.createStatement( + ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY); + JDBC.assertDrainResults( + scrollStmt.executeQuery("select * from d2354"), + 40000); + + stmt.close(); + scrollStmt.close(); + jarConn.close(); + + // Cleanup. Shut down the database and delete it. + shutdownDB("jdbc:derby:jar:(d2354db.jar)d2354db;shutdown=true"); + removeFiles(new String[] { + System.getProperty("derby.system.home") + "/d2354db.jar" + }); + } + + /** + * Create a database in a jar for use in {@code testSpillHashToDisk}. + */ + private void createDerby2354Database() throws SQLException { + // First create an ordinary database with a table. + Connection conn = + DriverManager.getConnection("jdbc:derby:d2354db;create=true"); + conn.setAutoCommit(false); + Statement s = conn.createStatement(); + s.execute("create table d2354 (x varchar(100))"); + s.close(); + + // Insert 40000 unique values into the table. The values should be + // unique so that they all occupy an entry in the hash table used by + // the DISTINCT query in the test, and thereby increase the likelihood + // of spilling to disk. + PreparedStatement insert = + conn.prepareStatement( + "insert into d2354 values ? || " + + "'some extra data to increase the size of the table'"); + for (int i = 0; i < 40000; i++) { + insert.setInt(1, i); + insert.executeUpdate(); + } + insert.close(); + + conn.commit(); + conn.close(); + + // Shut down the database and archive it in a jar file. + shutdownDB("jdbc:derby:d2354db;shutdown=true"); + + createStatement().execute( + "CALL CREATEARCHIVE('d2354db.jar', 'd2354db', 'd2354db')"); + + // Clean up the original database directory. We don't need it anymore + // now that we have archived it in a jar file. + removeDirectory( + new File(System.getProperty("derby.system.home") + "/d2354db")); + } protected static Test baseSuite(String name) { TestSuite suite = new TestSuite(name); Modified: db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/BaseTestCase.java URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/BaseTestCase.java?rev=1389677&r1=1389676&r2=1389677&view=diff ============================================================================== --- db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/BaseTestCase.java (original) +++ db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/BaseTestCase.java Tue Sep 25 01:35:57 2012 @@ -700,6 +700,15 @@ public abstract class BaseTestCase { DropDatabaseSetup.removeDirectory(dir); } + + /** + * Remove all the files in the list + * @param list the list contains all the files + */ + public static void removeFiles(String[] list) + { + DropDatabaseSetup.removeFiles(list); + } /** * Fail; attaching an exception for more detail on cause. Modified: db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/DropDatabaseSetup.java URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/DropDatabaseSetup.java?rev=1389677&r1=1389676&r2=1389677&view=diff ============================================================================== --- db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/DropDatabaseSetup.java (original) +++ db/derby/code/branches/10.7/java/testing/org/apache/derbyTesting/junit/DropDatabaseSetup.java Tue Sep 25 01:35:57 2012 @@ -118,4 +118,19 @@ class DropDatabaseSetup extends BaseTest BaseJDBCTestCase.assertDirectoryDeleted(dir); } + + /** + * Remove all the files in the list + * @param list the list of files that will be deleted + **/ + static void removeFiles(String[] list) { + for (int i = 0; i < list.length; i++) { + try { + File dfile = new File(list[i].toString()); + assertTrue(list[i].toString(), dfile.delete()); + } catch (IllegalArgumentException e) { + fail("open file error"); + } + } + } }