Return-Path: X-Original-To: apmail-db-derby-commits-archive@www.apache.org Delivered-To: apmail-db-derby-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id DE78310AAE for ; Thu, 11 Jul 2013 18:26:31 +0000 (UTC) Received: (qmail 82385 invoked by uid 500); 11 Jul 2013 18:26:31 -0000 Delivered-To: apmail-db-derby-commits-archive@db.apache.org Received: (qmail 82325 invoked by uid 500); 11 Jul 2013 18:26:29 -0000 Mailing-List: contact derby-commits-help@db.apache.org; run by ezmlm Precedence: bulk list-help: list-unsubscribe: List-Post: Reply-To: "Derby Development" List-Id: Delivered-To: mailing list derby-commits@db.apache.org Received: (qmail 82307 invoked by uid 99); 11 Jul 2013 18:26:26 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 11 Jul 2013 18:26:26 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 11 Jul 2013 18:26:24 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 6E59A238889B; Thu, 11 Jul 2013 18:26:04 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1502319 - in /db/derby/code/trunk/java: engine/org/apache/derby/impl/services/daemon/ testing/org/apache/derbyTesting/functionTests/tests/upgradeTests/ testing/org/apache/derbyTesting/functionTests/tests/upgradeTests/helpers/ Date: Thu, 11 Jul 2013 18:26:04 -0000 To: derby-commits@db.apache.org From: mikem@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20130711182604.6E59A238889B@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: mikem Date: Thu Jul 11 18:26:03 2013 New Revision: 1502319 URL: http://svn.apache.org/r1502319 Log: DERBY-6283 indexStat daemon processing tables over and over even when there are no changes in the tables in soft upgraded database. Changed system to always drop orphaned stats during update statistics call. Without this change soft upgraded systems running on 10.8 or higher derby software, that had an orphaned statistic would spin forever in the index stat daemon due to the same problem fixed by DERBY-5680 for hard upgraded databases. Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/services/daemon/IndexStatisticsDaemonImpl.java db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/upgradeTests/Changes10_9.java db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/upgradeTests/helpers/DisposableIndexStatistics.java Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/services/daemon/IndexStatisticsDaemonImpl.java URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/services/daemon/IndexStatisticsDaemonImpl.java?rev=1502319&r1=1502318&r2=1502319&view=diff ============================================================================== --- db/derby/code/trunk/java/engine/org/apache/derby/impl/services/daemon/IndexStatisticsDaemonImpl.java (original) +++ db/derby/code/trunk/java/engine/org/apache/derby/impl/services/daemon/IndexStatisticsDaemonImpl.java Thu Jul 11 18:26:03 2013 @@ -414,16 +414,20 @@ public class IndexStatisticsDaemonImpl ConglomerateDescriptor[] cds, boolean asBackgroundTask) throws StandardException { - final boolean identifyDisposableStats = - (cds == null && skipDisposableStats); + + // can only properly identify disposable stats if cds == null, + // which means we are processing all indexes on the conglomerate. + final boolean identifyDisposableStats = (cds == null); + // Fetch descriptors if we're updating statistics for all indexes. if (cds == null) { cds = td.getConglomerateDescriptors(); } + // Extract/derive information from the table descriptor - long[] conglomerateNumber = new long[cds.length]; - ExecIndexRow[] indexRow = new ExecIndexRow[cds.length]; - UUID[] objectUUID = new UUID[cds.length]; + long[] conglomerateNumber = new long[cds.length]; + ExecIndexRow[] indexRow = new ExecIndexRow[cds.length]; + TransactionController tc = lcc.getTransactionExecute(); ConglomerateController heapCC = @@ -434,6 +438,14 @@ public class IndexStatisticsDaemonImpl ? TransactionController.ISOLATION_READ_UNCOMMITTED : TransactionController.ISOLATION_REPEATABLE_READ ); + + + // create a list of indexes that should have statistics, by looking + // at all indexes on the conglomerate, and conditionally skipping + // unique single column indexes. This set is the "non disposable + // stat list". + UUID[] non_disposable_objectUUID = new UUID[cds.length]; + try { for (int i = 0; i < cds.length; i++) @@ -444,7 +456,9 @@ public class IndexStatisticsDaemonImpl conglomerateNumber[i] = -1; continue; } + IndexRowGenerator irg = cds[i].getIndexDescriptor(); + // Skip single-column unique indexes unless we're told not to, // or we are running in soft-upgrade-mode on a pre 10.9 db. if (skipDisposableStats) { @@ -454,9 +468,11 @@ public class IndexStatisticsDaemonImpl } } - conglomerateNumber[i] = cds[i].getConglomerateNumber(); - - objectUUID[i] = cds[i].getUUID(); + // at this point have found a stat for an existing + // index which is not a single column unique index, add it + // to the list of "non disposable stats" + conglomerateNumber[i] = cds[i].getConglomerateNumber(); + non_disposable_objectUUID[i] = cds[i].getUUID(); indexRow[i] = irg.getNullIndexRow( td.getColumnDescriptorList(), @@ -468,23 +484,39 @@ public class IndexStatisticsDaemonImpl heapCC.close(); } - // Check for disposable statistics if we have the required information. + // Check for and drop disposable statistics if we have the required + // information. + // // Note that the algorithm would drop valid statistics entries if // working on a subset of the table conglomerates/indexes. + // The above loop has populated "cds" with only existing indexes that + // are not single column unique. + if (identifyDisposableStats) { + + // Note this loop is not controlled by the skipDisposableStats + // flag. The above loop controls if we drop single column unique + // index stats or not. In all cases we are going to drop + // stats with no associated index (orphaned stats). + + List existingStats = td.getStatistics(); StatisticsDescriptor[] stats = (StatisticsDescriptor[]) existingStats.toArray( new StatisticsDescriptor[existingStats.size()]); + // For now we know that disposable stats only exist in two cases, // and that we'll only get one match for both of them per table: // a) orphaned statistics entries (i.e. DERBY-5681) // b) single-column primary keys + // + // This loop looks for statistic entries to delete. It deletes + // those entries that don't have a matching conglomerate in the for (int si=0; si < stats.length; si++) { UUID referencedIndex = stats[si].getReferenceID(); boolean isValid = false; for (int ci=0; ci < conglomerateNumber.length; ci++) { - if (referencedIndex.equals(objectUUID[ci])) { + if (referencedIndex.equals(non_disposable_objectUUID[ci])) { isValid = true; break; } @@ -518,7 +550,7 @@ public class IndexStatisticsDaemonImpl // [x][0] = conglomerate number, [x][1] = start time, [x][2] = stop time long[][] scanTimes = new long[conglomerateNumber.length][3]; - int sci = 0; + int sci = 0; for (int indexNumber = 0; indexNumber < conglomerateNumber.length; indexNumber++) @@ -535,10 +567,11 @@ public class IndexStatisticsDaemonImpl scanTimes[sci][0] = conglomerateNumber[indexNumber]; scanTimes[sci][1] = System.currentTimeMillis(); + // Subtract one for the RowLocation added for indexes. - int numCols = indexRow[indexNumber].nColumns() - 1; - long[] cardinality = new long[numCols]; - KeyComparator cmp = new KeyComparator(indexRow[indexNumber]); + int numCols = indexRow[indexNumber].nColumns() - 1; + long[] cardinality = new long[numCols]; + KeyComparator cmp = new KeyComparator(indexRow[indexNumber]); /* Read uncommitted, with record locking. Actually CS store may not hold record locks */ @@ -605,7 +638,8 @@ public class IndexStatisticsDaemonImpl int retries = 0; while (true) { try { - writeUpdatedStats(lcc, td, objectUUID[indexNumber], + writeUpdatedStats(lcc, td, + non_disposable_objectUUID[indexNumber], cmp.getRowCount(), cardinality, asBackgroundTask); break; } catch (StandardException se) { Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/upgradeTests/Changes10_9.java URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/upgradeTests/Changes10_9.java?rev=1502319&r1=1502318&r2=1502319&view=diff ============================================================================== --- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/upgradeTests/Changes10_9.java (original) +++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/upgradeTests/Changes10_9.java Thu Jul 11 18:26:03 2013 @@ -1003,7 +1003,9 @@ public class Changes10_9 extends Upgrade // version of Derby used to create the database. Some older // versions of Derby contained a bug and lacked optimizations, // causing the number of statistics entries to increase. - dis.assertStatsCount(false); + // Just after creation and before any update statistics expect + // all stats to exist. + dis.assertStatsCount(false, false); break; } // boot with new version and soft-upgrade @@ -1016,19 +1018,28 @@ public class Changes10_9 extends Upgrade ps.setString(1, tables[i]); ps.executeUpdate(); } - dis.assertStatsCount(false); + + // After soft upgrade and update statistics expect the + // orphaned index entry to be deleted, but the "unneeded + // disposable entries" are only deleted after hard upgrade. + dis.assertStatsCount(true, false); break; } // soft-downgrade: boot with old version after soft-upgrade case PH_POST_SOFT_UPGRADE: { - dis.assertStatsCount(false); + + // expect no change in entries on downgrade, should be same + // as they were in soft upgrade. + dis.assertStatsCount(true, false); break; } // boot with new version and hard-upgrade case PH_HARD_UPGRADE: { - dis.assertStatsCount(false); + // expect no change in entries on upgrade before update + // statistics. + dis.assertStatsCount(true, false); PreparedStatement ps = prepareStatement(updateStatsSQL); String[] tables = dis.getTableNames(); for (int i=0; i < tables.length; i++) { @@ -1038,7 +1049,7 @@ public class Changes10_9 extends Upgrade // Confirm that we disposed of the statistics that were added // due to a bug or simply not needed by Derby. try { - dis.assertStatsCount(true); + dis.assertStatsCount(true, true); } finally { for (int i=0; i < tables.length; i++) { dropTable(tables[i]); @@ -1049,5 +1060,4 @@ public class Changes10_9 extends Upgrade } } } - } Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/upgradeTests/helpers/DisposableIndexStatistics.java URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/upgradeTests/helpers/DisposableIndexStatistics.java?rev=1502319&r1=1502318&r2=1502319&view=diff ============================================================================== --- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/upgradeTests/helpers/DisposableIndexStatistics.java (original) +++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/upgradeTests/helpers/DisposableIndexStatistics.java Thu Jul 11 18:26:03 2013 @@ -153,7 +153,8 @@ public class DisposableIndexStatistics { // Several valid states here, use a relaxed range check. int max = getNumTotalPossibleStats(); - int min = max - getNumDisposableStats(); + int min = max - (getNumNotNeededDisposableStats() + + getNumOrphanedDisposableStats()); int cur = getAllRelevantStats(null); Assert.assertTrue("cur=" + cur + ", min=" + min, cur >= min); Assert.assertTrue("cur=" + cur + ", max=" + max, cur <= max); @@ -207,21 +208,35 @@ public class DisposableIndexStatistics { /** * Asserts the number of statistics entries for all relevant tables. * - * @param disposedOf tells if the disposable statistics entries are - * expected to have been removed at this point + * @param orphaned_disposedOf tells if the orphaned disposable statistics + * entries are expected to have been removed at this point + * @param notneeded_disposedOf tells if the unneeded statistics (like + * single column unique key indexes) are expected to have been + * removed at this point */ - public void assertStatsCount(boolean disposedOf) + public void assertStatsCount( + boolean orphaned_disposedOf, + boolean notneeded_disposedOf) throws SQLException { + int expected = getNumTotalPossibleStats(); - // Adjust expected count if the disposable stats should be gone. - if (disposedOf) { - expected -= getNumDisposableStats(); - } else if (!hasDerby5681Bug(oldVersion)) { + + if (!hasDerby5681Bug(oldVersion)) { // Here we correct for the orphaned statistics entry, but not for // entries that are considered extraneous by newer releases (for // instance statistics for single column unique indexes). - expected--; + expected -= getNumOrphanedDisposableStats(); + + } else if (orphaned_disposedOf) { + // if the bug exists and we got rid of the orphaned stat. + expected -= getNumOrphanedDisposableStats(); + } + + // Adjust expected count if the disposable stats should be gone. + if (notneeded_disposedOf) { + expected -= getNumNotNeededDisposableStats(); } + // used for reporting only ArrayList entries = new ArrayList(); @@ -279,8 +294,12 @@ public class DisposableIndexStatistics { } /** Number of disposable statistics entries. */ - public static int getNumDisposableStats() { - return 3; + public static int getNumNotNeededDisposableStats() { + return 2; + } + + public static int getNumOrphanedDisposableStats() { + return 1; } /**