db-derby-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mi...@apache.org
Subject svn commit: r179319 - in /incubator/derby/code/trunk/java: engine/org/apache/derby/impl/store/raw/data/ testing/org/apache/derbyTesting/functionTests/master/ testing/org/apache/derbyTesting/functionTests/tests/store/ testing/org/apache/derbyTesting/functionTests/tests/storetests/
Date Wed, 01 Jun 2005 00:10:38 GMT
Author: mikem
Date: Tue May 31 17:10:37 2005
New Revision: 179319

URL: http://svn.apache.org/viewcvs?rev=179319&view=rev
Log:
More tests and a fix for a problem shown up by new tests for online
compress.  Also deleting obsolete test, functionality has been moved
into the .java version of the test.


Removed:
    incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/storetests/onlineCompressTable.sql
Modified:
    incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/AllocExtent.java
    incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/AllocationCache.java
    incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/master/OnlineCompressTest.out
    incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest.java
    incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest_derby.properties

Modified: incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/AllocExtent.java
URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/AllocExtent.java?rev=179319&r1=179318&r2=179319&view=diff
==============================================================================
--- incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/AllocExtent.java
(original)
+++ incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/AllocExtent.java
Tue May 31 17:10:37 2005
@@ -423,6 +423,17 @@
 		setExtentFreePageStatus(true);
 	}
 
+    /**
+     * Compress free pages at end of this extent.
+     * <p>
+     * Search backward from end of extent and prepare data structures
+     * to return pages at end of extent to the OS. Returns the lowest
+     * page that can be returned to the OS.
+     * <p>
+     *
+	 * @return Return bit of page where page and all those that follow can
+     *         be returned to the OS.
+     **/
     protected long compressPages()
     {
         int compress_bitnum = -1;
@@ -432,6 +443,8 @@
             if (freePages.isSet(i))
             {
                 freePages.clear(i);
+                unFilledPages.clear(i);
+
                 compress_bitnum = i;
             }
             else
@@ -449,7 +462,6 @@
         {
             return(-1);
         }
-
     }
 
 	protected long getExtentEnd()

Modified: incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/AllocationCache.java
URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/AllocationCache.java?rev=179319&r1=179318&r2=179319&view=diff
==============================================================================
--- incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/AllocationCache.java
(original)
+++ incubator/derby/code/trunk/java/engine/org/apache/derby/impl/store/raw/data/AllocationCache.java
Tue May 31 17:10:37 2005
@@ -228,12 +228,19 @@
                 currAllocPages = extents[i].getAllocatedPageCount();
 				numAllocatedPages += currAllocPages;
                 numUnfilledPages += extents[i].getUnfilledPageCount();
-                numFreePages += (extents[i].getTotalPageCount() - currAllocPages);
+                numFreePages += 
+                    (extents[i].getTotalPageCount() - currAllocPages);
             }
 
             if (SanityManager.DEBUG)
+            {
                 SanityManager.ASSERT(numUnfilledPages <= numAllocatedPages,
-                    "more unfilled pages than allocated pages on extent ");
+                    "more unfilled pages than allocated pages on extent[" + i +
+                    "], " +
+                    "numUnfilledPages = "    + numUnfilledPages + 
+                    ", numAllocatedPages = " + numAllocatedPages +
+                    ", numFreePages = "      + numFreePages);
+            }
 		}
         return new SpaceInformation(
             numAllocatedPages,

Modified: incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/master/OnlineCompressTest.out
URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/master/OnlineCompressTest.out?rev=179319&r1=179318&r2=179319&view=diff
==============================================================================
--- incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/master/OnlineCompressTest.out
(original)
+++ incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/master/OnlineCompressTest.out
Tue May 31 17:10:37 2005
@@ -40,3 +40,35 @@
 Executing test: begin checkPurgePhase4000 row test, create = false.
 Executing test: end checkPurgePhase4000 row test.
 Ending test: test1
+Beginning test: test3
+Executing test: begin deleteAllRows,1 row test, create = true.
+Executing test: no delete case complete.
+Executing test: delete all rows case succeeded.
+Executing test: end deleteAllRows,1 row test.
+Executing test: begin deleteAllRows,1 row test, create = false.
+Executing test: no delete case complete.
+Executing test: delete all rows case succeeded.
+Executing test: end deleteAllRows,1 row test.
+Executing test: begin checkPurgePhase1 row test, create = false.
+Executing test: end checkPurgePhase1 row test.
+Executing test: begin deleteAllRows,2 row test, create = true.
+Executing test: no delete case complete.
+Executing test: delete all rows case succeeded.
+Executing test: end deleteAllRows,2 row test.
+Executing test: begin deleteAllRows,2 row test, create = false.
+Executing test: no delete case complete.
+Executing test: delete all rows case succeeded.
+Executing test: end deleteAllRows,2 row test.
+Executing test: begin checkPurgePhase2 row test, create = false.
+Executing test: end checkPurgePhase2 row test.
+Executing test: begin deleteAllRows,50 row test, create = true.
+Executing test: no delete case complete.
+Executing test: delete all rows case succeeded.
+Executing test: end deleteAllRows,50 row test.
+Executing test: begin deleteAllRows,50 row test, create = false.
+Executing test: no delete case complete.
+Executing test: delete all rows case succeeded.
+Executing test: end deleteAllRows,50 row test.
+Executing test: begin checkPurgePhase50 row test, create = false.
+Executing test: end checkPurgePhase50 row test.
+Ending test: test3

Modified: incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest.java
URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest.java?rev=179319&r1=179318&r2=179319&view=diff
==============================================================================
--- incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest.java
(original)
+++ incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest.java
Tue May 31 17:10:37 2005
@@ -164,7 +164,6 @@
      * This schema fits 25 rows per page
      * <p>
      *
-	 * @return The identifier to be used to open the conglomerate later.
      *
      * @param conn          Connection to use for sql execution.
      * @param create_table  If true, create new table - otherwise load into
@@ -241,6 +240,140 @@
         conn.commit();
     }
 
+    /**
+     * Create and load a table with long columns and long rows.
+     * <p>
+     * If create_table is set creates a test data table with indexes.
+     * Loads num_rows into the table.  This table defaults to 32k page size.
+     * <p>
+     * schema of table:
+     *     keycol   int, 
+     *     longcol1 clob(200k),
+     *     longrow1 varchar(10000),
+     *     longrow2 varchar(10000),
+     *     longrow3 varchar(10000),
+     *     longrow4 varchar(10000),
+     *     indcol1  int, 
+     *     indcol2  int, 
+     *     indcol3  int, 
+     *     data1    varchar(2000), 
+     *     data2    varchar(2000)
+     *     longrow5 varchar(10000),
+     *     longrow6 varchar(10000),
+     *     longrow7 varchar(10000),
+     *     longrow8 varchar(10000),
+     *     longcol2 clob(200k),
+     *
+     *
+     * @param conn          Connection to use for sql execution.
+     * @param create_table  If true, create new table - otherwise load into
+     *                      existing table.
+     * @param tblname       table to use.
+     * @param num_rows      number of rows to add to the table.
+     *
+	 * @exception  StandardException  Standard exception policy.
+     **/
+    private void createAndLoadLongTable(
+    Connection  conn,
+    boolean     create_table,
+    String      tblname,
+    int         num_rows)
+        throws SQLException
+    {
+        if (create_table)
+        {
+            Statement s = conn.createStatement();
+
+            s.execute(
+                "create table " + tblname + 
+                " (keycol   int, longcol1 clob(200k), longrow1 varchar(10000), longrow2 varchar(10000),
longrow3 varchar(10000), longrow4 varchar(10000), indcol1  int, indcol2  int, indcol3  int,
data1    varchar(2000), data2    varchar(2000), longrow5 varchar(10000), longrow6 varchar(10000),
longrow7 varchar(10000), longrow8 varchar(10000), longcol2 clob(200k))");
+            s.close();
+        }
+
+        PreparedStatement insert_stmt = 
+            conn.prepareStatement(
+                "insert into " + tblname + " values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?)");
+
+        char[]  data1_data = new char[500];
+        char[]  data2_data = new char[500];
+
+        for (int i = 0; i < data1_data.length; i++)
+        {
+            data1_data[i] = 'a';
+            data2_data[i] = 'b';
+        }
+        String  data1_str = new String(data1_data);
+        String  data2_str = new String(data2_data);
+
+        // some data to force row to be bigger than a page, ie. long row
+        char[] data3_data = new char[10000];
+        char[] data4_data = new char[10000];
+
+        for (int i = 0; i < data3_data.length; i++)
+        {
+            data3_data[i] = 'c';
+            data4_data[i] = 'd';
+        }
+        String  data3_str = new String(data3_data);
+        String  data4_str = new String(data4_data);
+
+        // some data for the long columns
+        char[] data5_data = new char[200000];
+        char[] data6_data = new char[200000];
+
+        for (int i = 0; i < data5_data.length; i++)
+        {
+            data5_data[i] = 'e';
+            data6_data[i] = 'f';
+        }
+
+        String  data5_str = new String(data5_data);
+        String  data6_str = new String(data6_data);
+
+        for (int i = 0; i < num_rows; i++)
+        {
+            insert_stmt.setInt(1, i);               // keycol
+            insert_stmt.setString(2, data5_str);    // longcol1
+            insert_stmt.setString(3, data3_str);    // longrow1
+            insert_stmt.setString(4, data3_str);    // longrow2
+            insert_stmt.setString(5, data3_str);    // longrow3
+            insert_stmt.setString(6, data3_str);    // longrow4
+            insert_stmt.setInt(7, i * 10);          // indcol1
+            insert_stmt.setInt(8, i * 100);         // indcol2
+            insert_stmt.setInt(9, -i);              // indcol3
+            insert_stmt.setString(10, data1_str);   // data1_data
+            insert_stmt.setString(11, data2_str);   // data2_data
+            insert_stmt.setString(12, data4_str);   // longrow5
+            insert_stmt.setString(13, data4_str);   // longrow6
+            insert_stmt.setString(14, data4_str);   // longrow7
+            insert_stmt.setString(15, data4_str);   // longrow8
+            insert_stmt.setString(16, data5_str);   // longcol2
+
+            insert_stmt.execute();
+        }
+
+        if (create_table)
+        {
+            Statement s = conn.createStatement();
+
+            s.execute(
+                "create index " + tblname + "_idx_keycol on " + tblname +
+                    "(keycol)");
+            s.execute(
+                "create index " + tblname + "_idx_indcol1 on " + tblname +
+                    "(indcol1)");
+            s.execute(
+                "create index " + tblname + "_idx_indcol2 on " + tblname +
+                    "(indcol2)");
+            s.execute(
+                "create unique index " + tblname + "_idx_indcol3 on " + tblname +
+                    "(indcol3)");
+            s.close();
+        }
+
+        conn.commit();
+    }
+
     private void executeQuery(
     Connection  conn,
     String      stmt_str,
@@ -285,6 +418,7 @@
     private void deleteAllRows(
     Connection  conn,
     boolean     create_table,
+    boolean     long_table,
     String      schemaName,
     String      table_name,
     int         num_rows) 
@@ -295,7 +429,10 @@
                 create_table + ".");
 
 
-        createAndLoadTable(conn, create_table, table_name, num_rows);
+        if (long_table)
+            createAndLoadLongTable(conn, create_table, table_name, num_rows);
+        else
+            createAndLoadTable(conn, create_table, table_name, num_rows);
 
         if (verbose)
             testProgress("Calling compress.");
@@ -374,6 +511,7 @@
     private void checkPurgePhase(
     Connection  conn,
     boolean     create_table,
+    boolean     long_table,
     String      schemaName,
     String      table_name,
     int         num_rows) 
@@ -383,7 +521,10 @@
             "begin checkPurgePhase" + num_rows + " row test, create = " + 
                 create_table + ".");
 
-        createAndLoadTable(conn, create_table, table_name, num_rows);
+        if (long_table)
+            createAndLoadLongTable(conn, create_table, table_name, num_rows);
+        else
+            createAndLoadTable(conn, create_table, table_name, num_rows);
 
         // dump_table(conn, schemaName, table_name, false);
 
@@ -501,7 +642,10 @@
         // will abort transaction.
         executeQuery(conn, "delete from " + table_name, true);
         callCompress(conn, "APP", table_name, true, true, true, true);
-        createAndLoadTable(conn, create_table, table_name, num_rows);
+        if (long_table)
+            createAndLoadLongTable(conn, create_table, table_name, num_rows);
+        else
+            createAndLoadTable(conn, create_table, table_name, num_rows);
         conn.commit();
         executeQuery(conn, "delete from " + table_name, false);
 
@@ -594,7 +738,7 @@
     }
 
     /**
-     * Test 1 alloc page test cases.
+     * Test 1 - various # page tests, regular row/columns
      * <p>
      * perform a number of insert/delete/compress operations on a variety
      * of sized tables, use space allocation information to verify that
@@ -628,15 +772,15 @@
         {
             // first create new table and run the tests.
             deleteAllRows(
-                conn, true, "APP", table_name, test_cases[i]);
+                conn, true, false, "APP", table_name, test_cases[i]);
 
             // now rerun tests on existing table, which had all rows deleted
             // and truncated.
             deleteAllRows(
-                conn, false, "APP", table_name, test_cases[i]);
+                conn, false, false, "APP", table_name, test_cases[i]);
 
             checkPurgePhase(
-                conn, false, "APP", table_name, test_cases[i]);
+                conn, false, false, "APP", table_name, test_cases[i]);
 
             executeQuery(conn, "drop table " + table_name, true);
         }
@@ -645,88 +789,80 @@
     }
 
     /**
-     * Purge of uncommitted deletes should not do anything.
+     * Test 2 - check transaction roll backs
      * <p>
-     * In the same transaction insert a number of rows, delete them all
-     * and then run the purge operation.  The purge operation should find
-     * the rows deleted but not do anything with them as the transaction
-     * has not committed.
+     *
      **/
     private void test2(
     Connection  conn,
     String      test_name,
-    String      table_name,
-    int         num_rows)
+    String      table_name)
         throws SQLException 
     {
-        beginTest(conn, test_name);
+    }
 
-        createAndLoadTable(conn, true, table_name, num_rows);
 
-        // Purge pass on non-committed deleted rows should do nothing.  
 
-        int[] ret_before = getSpaceInfo(conn, "APP", table_name, false);
+    /**
+     * Test 3 - various # page tests, long row and long columns
+     * <p>
+     * perform a number of insert/delete/compress operations on a variety
+     * of sized tables, use space allocation information to verify that
+     * compression is happening and use consistency checker to verify that
+     * tables and indexes are all valid following the operations.
+     * <p>
+     * loop through testing interesting row count cases.  The cases are
+     * 0    rows  - basic edge case
+     * 1    row   - another edge case
+     * 100  rows  - ~50 meg table
+     * 4000 rows  - ~2 gig table
+     *
+     * note that row numbers greater than 4000 may lead to lock escalation
+     * issues, if queries like "delete from x" are used to delete all the 
+     * rows.
+     *
+     * <p>
+     *
+     **/
+    private void test3(
+    Connection  conn,
+    String      test_name,
+    String      table_name)
+        throws SQLException 
+    {
+        beginTest(conn, test_name);
 
-        // Calling compress with just the "purge" pass option, no commit called.
-        callCompress(conn, "APP", table_name, true, false, false, false);
-        int[] ret_after  = getSpaceInfo(conn, "APP", table_name, false);
+        // note that 500 rows took 30 minutes on a ~1.5 ghz laptop
+        int[] test_cases = {1, 2, 50};
 
-        if (ret_after[SPACE_INFO_NUM_ALLOC] != ret_before[SPACE_INFO_NUM_ALLOC])
-        {
-            log_wrong_count(
-                "Expected no alloc page change.", 
-                table_name, num_rows, 
-                ret_before[SPACE_INFO_NUM_ALLOC], 
-                ret_after[SPACE_INFO_NUM_ALLOC],
-                ret_before, ret_after);
-        }
-        if (ret_after[SPACE_INFO_NUM_FREE] != ret_before[SPACE_INFO_NUM_FREE])
+        for (int i = 0; i < test_cases.length; i++)
         {
-            log_wrong_count(
-                "Expected no alloc page change.", 
-                table_name, num_rows, 
-                ret_before[SPACE_INFO_NUM_ALLOC], 
-                ret_after[SPACE_INFO_NUM_ALLOC],
-                ret_before, ret_after);
-        }
+            // first create new table and run the tests.
+            deleteAllRows(
+                conn, true, true, "APP", table_name, test_cases[i]);
 
-        // since table was just loaded there a defragment pass also should
-        // not find anything to do.
-        
-        // Calling compress with just the "defragment" option, no commit called.
-        callCompress(conn, "APP", table_name, false, true, false, false);
-        ret_after  = getSpaceInfo(conn, "APP", table_name, false);
+            // now rerun tests on existing table, which had all rows deleted
+            // and truncated.
+            deleteAllRows(
+                conn, false, true, "APP", table_name, test_cases[i]);
 
-        if (ret_after[SPACE_INFO_NUM_ALLOC] != ret_before[SPACE_INFO_NUM_ALLOC])
-        {
-            log_wrong_count(
-                "Expected no alloc page change.", 
-                table_name, num_rows, 
-                ret_before[SPACE_INFO_NUM_ALLOC], 
-                ret_after[SPACE_INFO_NUM_ALLOC],
-                ret_before, ret_after);
-        }
-        if (ret_after[SPACE_INFO_NUM_FREE] != ret_before[SPACE_INFO_NUM_FREE])
-        {
-            log_wrong_count(
-                "Expected no alloc page change.", 
-                table_name, num_rows, 
-                ret_before[SPACE_INFO_NUM_ALLOC], 
-                ret_after[SPACE_INFO_NUM_ALLOC],
-                ret_before, ret_after);
-        }
+            checkPurgePhase(
+                conn, false, true, "APP", table_name, test_cases[i]);
 
-        executeQuery(conn, "drop table " + table_name, true);
+            executeQuery(conn, "drop table " + table_name, true);
+        }
 
         endTest(conn, test_name);
     }
 
 
+
     public void testList(Connection conn)
         throws SQLException
     {
         test1(conn, "test1", "TEST1");
         // test2(conn, "test2", "TEST2", 10000);
+        test3(conn, "test3", "TEST3");
     }
 
     public static void main(String[] argv) 

Modified: incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest_derby.properties
URL: http://svn.apache.org/viewcvs/incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest_derby.properties?rev=179319&r1=179318&r2=179319&view=diff
==============================================================================
--- incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest_derby.properties
(original)
+++ incubator/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/store/OnlineCompressTest_derby.properties
Tue May 31 17:10:37 2005
@@ -1,8 +1,8 @@
 usedefaults=true
 
 #derby.storage.userLockTrace=true
-#derby.locks.deadlockTimeout=1
-#derby.locks.waitTimeout=3
+derby.locks.deadlockTimeout=1
+derby.locks.waitTimeout=3
 
 derby.infolog.append=true
 



Mime
View raw message