Return-Path: Delivered-To: apmail-db-derby-commits-archive@www.apache.org Received: (qmail 65972 invoked from network); 7 Oct 2006 01:33:39 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (209.237.227.199) by minotaur.apache.org with SMTP; 7 Oct 2006 01:33:39 -0000 Received: (qmail 31737 invoked by uid 500); 7 Oct 2006 01:33:38 -0000 Delivered-To: apmail-db-derby-commits-archive@db.apache.org Received: (qmail 31661 invoked by uid 500); 7 Oct 2006 01:33:37 -0000 Mailing-List: contact derby-commits-help@db.apache.org; run by ezmlm Precedence: bulk list-help: list-unsubscribe: List-Post: Reply-To: "Derby Development" List-Id: Delivered-To: mailing list derby-commits@db.apache.org Received: (qmail 31603 invoked by uid 99); 7 Oct 2006 01:33:37 -0000 Received: from idunn.apache.osuosl.org (HELO idunn.apache.osuosl.org) (140.211.166.84) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 06 Oct 2006 18:33:37 -0700 X-ASF-Spam-Status: No, hits=-9.4 required=5.0 tests=ALL_TRUSTED,NO_REAL_NAME Received: from [140.211.166.113] ([140.211.166.113:61859] helo=eris.apache.org) by idunn.apache.osuosl.org (ecelerity 2.1.1.8 r(12930)) with ESMTP id 4F/50-24193-DE307254 for ; Fri, 06 Oct 2006 18:33:34 -0700 Received: by eris.apache.org (Postfix, from userid 65534) id 773711A981A; Fri, 6 Oct 2006 18:33:30 -0700 (PDT) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r453832 - /db/derby/code/branches/10.1/java/engine/org/apache/derby/iapi/store/access/DiskHashtable.java Date: Sat, 07 Oct 2006 01:33:30 -0000 To: derby-commits@db.apache.org From: mikem@apache.org X-Mailer: svnmailer-1.1.0 Message-Id: <20061007013330.773711A981A@eris.apache.org> X-Spam-Rating: minotaur.apache.org 1.6.2 0/1000/N Author: mikem Date: Fri Oct 6 18:33:29 2006 New Revision: 453832 URL: http://svn.apache.org/viewvc?view=rev&rev=453832 Log: DERBY-1939 Adding an ASSERT to catch when the problem happens, rather than later as a side effect. The problem is that template created to manipulate the hash base table gets a null for the HeapRowLocation, later when that template is used store interprets the null as an object has been stored on disk and reads it as if it is a self described object rather than a HeapRowLocation. This just adds the assert, it does not fix the issue. Also added some comments that helped me understand what was going on. Modified: db/derby/code/branches/10.1/java/engine/org/apache/derby/iapi/store/access/DiskHashtable.java Modified: db/derby/code/branches/10.1/java/engine/org/apache/derby/iapi/store/access/DiskHashtable.java URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.1/java/engine/org/apache/derby/iapi/store/access/DiskHashtable.java?view=diff&rev=453832&r1=453831&r2=453832 ============================================================================== --- db/derby/code/branches/10.1/java/engine/org/apache/derby/iapi/store/access/DiskHashtable.java (original) +++ db/derby/code/branches/10.1/java/engine/org/apache/derby/iapi/store/access/DiskHashtable.java Fri Oct 6 18:33:29 2006 @@ -28,17 +28,18 @@ import org.apache.derby.iapi.services.io.FormatableBitSet; import org.apache.derby.iapi.types.DataValueDescriptor; import org.apache.derby.iapi.types.SQLInteger; -import org.apache.derby.impl.store.access.heap.HeapRowLocation; import org.apache.derby.iapi.types.RowLocation; import org.apache.derby.iapi.services.context.ContextService; import org.apache.derby.iapi.sql.conn.LanguageConnectionContext; +import org.apache.derby.iapi.services.sanity.SanityManager; /** - * This class is used by BackingStoreHashtable when the BackingStoreHashtable must spill to disk. - * It implements the methods of a hash table: put, get, remove, elements, however it is not implemented - * as a hash table. In order to minimize the amount of unique code it is implemented using a Btree and a heap - * conglomerate. The Btree indexes the hash code of the row key. The actual key may be too long for - * our Btree implementation. + * This class is used by BackingStoreHashtable when the BackingStoreHashtable + * must spill to disk. It implements the methods of a hash table: put, get, + * remove, elements, however it is not implemented as a hash table. In order to + * minimize the amount of unique code it is implemented using a Btree and a + * heap conglomerate. The Btree indexes the hash code of the row key. The + * actual key may be too long for our Btree implementation. * * Created: Fri Jan 28 13:58:03 2005 * @@ -48,77 +49,126 @@ public class DiskHashtable { - private final long rowConglomerateId; - private ConglomerateController rowConglomerate; - private final long btreeConglomerateId; - private ConglomerateController btreeConglomerate; - private final DataValueDescriptor[] btreeRow; - private final int[] key_column_numbers; - private final boolean remove_duplicates; - private final TransactionController tc; - private final DataValueDescriptor[] row; - private final DataValueDescriptor[] scanKey = { new SQLInteger()}; - private int size; - private boolean keepStatistics; + private final long rowConglomerateId; + private ConglomerateController rowConglomerate; + private final long btreeConglomerateId; + private ConglomerateController btreeConglomerate; + private final DataValueDescriptor[] btreeRow; + private final int[] key_column_numbers; + private final boolean remove_duplicates; + private final TransactionController tc; + private final DataValueDescriptor[] row; + private final DataValueDescriptor[] scanKey = { new SQLInteger()}; + private int size; + private boolean keepStatistics; /** * Creates a new DiskHashtable instance. * * @param tc - * @param template An array of DataValueDescriptors that serves as a template for the rows. - * @param key_column_numbers The indexes of the key columns (0 based) - * @param remove_duplicates If true then rows with duplicate keys are removed - * @param keepAfterCommit If true then the hash table is kept after a commit + * @param template An array of DataValueDescriptors that + * serves as a template for the rows. + * @param key_column_numbers The indexes of the key columns (0 based) + * @param remove_duplicates If true then rows with duplicate keys are + * removed. + * @param keepAfterCommit If true then the hash table is kept after + * a commit */ - public DiskHashtable( TransactionController tc, - DataValueDescriptor[] template, - int[] key_column_numbers, - boolean remove_duplicates, - boolean keepAfterCommit) + public DiskHashtable( + TransactionController tc, + DataValueDescriptor[] template, + int[] key_column_numbers, + boolean remove_duplicates, + boolean keepAfterCommit) throws StandardException { - this.tc = tc; - this.key_column_numbers = key_column_numbers; - this.remove_duplicates = remove_duplicates; - LanguageConnectionContext lcc = (LanguageConnectionContext) - ContextService.getContextOrNull(LanguageConnectionContext.CONTEXT_ID); + this.tc = tc; + this.key_column_numbers = key_column_numbers; + this.remove_duplicates = remove_duplicates; + LanguageConnectionContext lcc = (LanguageConnectionContext) + ContextService.getContextOrNull( + LanguageConnectionContext.CONTEXT_ID); + keepStatistics = (lcc != null) && lcc.getRunTimeStatisticsMode(); - row = new DataValueDescriptor[ template.length]; + + // Create template row used for creating the conglomerate and + // fetching rows. + row = new DataValueDescriptor[template.length]; for( int i = 0; i < row.length; i++) + { row[i] = template[i].getNewNull(); - int tempFlags = keepAfterCommit ? (TransactionController.IS_TEMPORARY | TransactionController.IS_KEPT) - : TransactionController.IS_TEMPORARY; + + if (SanityManager.DEBUG) + { + // must have an object template for all cols in hash overflow. + SanityManager.ASSERT( + row[i] != null, + "Template for the hash table must have non-null object"); + } + } + + int tempFlags = + keepAfterCommit ? + (TransactionController.IS_TEMPORARY | + TransactionController.IS_KEPT) : + TransactionController.IS_TEMPORARY; - rowConglomerateId = tc.createConglomerate( "heap", - template, - (ColumnOrdering[]) null, - (Properties) null, - tempFlags); - rowConglomerate = tc.openConglomerate( rowConglomerateId, - keepAfterCommit, - TransactionController.OPENMODE_FORUPDATE, - TransactionController.MODE_TABLE, - TransactionController.ISOLATION_NOLOCK /* Single thread only */ ); + // create the "base" table of the hash overflow. + rowConglomerateId = + tc.createConglomerate( + "heap", + template, + (ColumnOrdering[]) null, + (Properties) null, + tempFlags); + + // open the "base" table of the hash overflow. + rowConglomerate = + tc.openConglomerate( + rowConglomerateId, + keepAfterCommit, + TransactionController.OPENMODE_FORUPDATE, + TransactionController.MODE_TABLE, + TransactionController.ISOLATION_NOLOCK/* Single thread only */); + + // create the index on the "hash" base table. The key of the index + // is the hash code of the row key. The second column is the + // RowLocation of the row in the "base" table of the hash overflow. + btreeRow = + new DataValueDescriptor[] + { new SQLInteger(), rowConglomerate.newRowLocationTemplate()}; - btreeRow = new DataValueDescriptor[] { new SQLInteger(), rowConglomerate.newRowLocationTemplate()}; Properties btreeProps = new Properties(); - btreeProps.put( "baseConglomerateId", String.valueOf( rowConglomerateId)); - btreeProps.put( "rowLocationColumn", "1"); - btreeProps.put( "allowDuplicates", "false"); // Because the row location is part of the key - btreeProps.put( "nKeyFields", "2"); // Include the row location column - btreeProps.put( "nUniqueColumns", "2"); // Include the row location column - btreeProps.put( "maintainParentLinks", "false"); - btreeConglomerateId = tc.createConglomerate( "BTREE", - btreeRow, - (ColumnOrdering[]) null, - btreeProps, - tempFlags); - - btreeConglomerate = tc.openConglomerate( btreeConglomerateId, - keepAfterCommit, - TransactionController.OPENMODE_FORUPDATE, - TransactionController.MODE_TABLE, - TransactionController.ISOLATION_NOLOCK /* Single thread only */ ); + + btreeProps.put("baseConglomerateId", + String.valueOf(rowConglomerateId)); + btreeProps.put("rowLocationColumn", + "1"); + btreeProps.put("allowDuplicates", + "false"); // Because the row location is part of the key + btreeProps.put("nKeyFields", + "2"); // Include the row location column + btreeProps.put("nUniqueColumns", + "2"); // Include the row location column + btreeProps.put("maintainParentLinks", + "false"); + btreeConglomerateId = + tc.createConglomerate( + "BTREE", + btreeRow, + (ColumnOrdering[]) null, + btreeProps, + tempFlags); + + // open the "index" of the hash overflow. + btreeConglomerate = + tc.openConglomerate( + btreeConglomerateId, + keepAfterCommit, + TransactionController.OPENMODE_FORUPDATE, + TransactionController.MODE_TABLE, + TransactionController.ISOLATION_NOLOCK /*Single thread only*/ ); + } // end of constructor public void close() throws StandardException @@ -134,49 +184,60 @@ * * @param row The row to be inserted. * - * @return true if the row was added, - * false if it was not added (because it was a duplicate and we are eliminating duplicates). + * @return true if the row was added, + * false if it was not added (because it was a duplicate and we + * are eliminating duplicates). * * @exception StandardException standard error policy */ - public boolean put( Object key, Object[] row) + public boolean put(Object key, Object[] row) throws StandardException { boolean isDuplicate = false; - if( remove_duplicates || keepStatistics) + if (remove_duplicates || keepStatistics) { // Go to the work of finding out whether it is a duplicate - isDuplicate = (getRemove( key, false, true) != null); - if( remove_duplicates && isDuplicate) + isDuplicate = (getRemove(key, false, true) != null); + if (remove_duplicates && isDuplicate) return false; } - rowConglomerate.insertAndFetchLocation( (DataValueDescriptor[]) row, (RowLocation) btreeRow[1]); + + // insert the row into the "base" conglomerate. + rowConglomerate.insertAndFetchLocation( + (DataValueDescriptor[]) row, (RowLocation) btreeRow[1]); + + // create index row from hashcode and rowlocation just inserted, and + // insert index row into index. btreeRow[0].setValue( key.hashCode()); btreeConglomerate.insert( btreeRow); - if( keepStatistics && !isDuplicate) + + if (keepStatistics && !isDuplicate) size++; + return true; + } // end of put /** * Get a row from the overflow structure. * - * @param key If the rows only have one key column then the key value. If there is more than one - * key column then a KeyHasher + * @param key If the rows only have one key column then the key value. + * If there is more than one key column then a KeyHasher * * @return null if there is no corresponding row, - * the row (DataValueDescriptor[]) if there is exactly one row with the key + * the row (DataValueDescriptor[]) if there is exactly one row + * with the key, or * a Vector of all the rows with the key if there is more than one. * * @exception StandardException */ - public Object get( Object key) + public Object get(Object key) throws StandardException { - return getRemove( key, false, false); + return getRemove(key, false, false); } - private Object getRemove( Object key, boolean remove, boolean existenceOnly) + private Object getRemove(Object key, boolean remove, boolean existenceOnly) throws StandardException { int hashCode = key.hashCode(); @@ -184,22 +245,27 @@ Object retValue = null; scanKey[0].setValue( hashCode); - ScanController scan = tc.openScan( btreeConglomerateId, - false, // do not hold - remove ? TransactionController.OPENMODE_FORUPDATE : 0, - TransactionController.MODE_TABLE, - TransactionController.ISOLATION_READ_UNCOMMITTED, - null, // Scan all the columns - scanKey, - ScanController.GE, - (Qualifier[][]) null, - scanKey, - ScanController.GT); + ScanController scan = + tc.openScan( + btreeConglomerateId, + false, // do not hold + remove ? TransactionController.OPENMODE_FORUPDATE : 0, + TransactionController.MODE_TABLE, + TransactionController.ISOLATION_READ_UNCOMMITTED, + null, // Scan all the columns + scanKey, + ScanController.GE, + (Qualifier[][]) null, + scanKey, + ScanController.GT); try { - while( scan.fetchNext( btreeRow)) + while (scan.fetchNext(btreeRow)) { - if( rowConglomerate.fetch( (RowLocation) btreeRow[1], row, (FormatableBitSet) null /* all columns */) + if (rowConglomerate.fetch( + (RowLocation) btreeRow[1], + row, + (FormatableBitSet) null /* all columns */) && rowMatches( row, key)) { if( existenceOnly) @@ -207,18 +273,29 @@ rowCount++; if( rowCount == 1) + { + // if there is only one matching row just return row. retValue = BackingStoreHashtable.cloneRow( row); + } else { + // if there is more than one row, return a vector of + // the rows. + // Vector v; if( rowCount == 2) { + // convert the "single" row retrieved from the + // first trip in the loop, to a vector with the + // first two rows. v = new Vector( 2); v.add( retValue); retValue = v; } else + { v = (Vector) retValue; + } v.add( BackingStoreHashtable.cloneRow( row)); } if( remove) @@ -241,8 +318,9 @@ } // end of getRemove - private boolean rowMatches( DataValueDescriptor[] row, - Object key) + private boolean rowMatches( + DataValueDescriptor[] row, + Object key) { if( key_column_numbers.length == 1) return row[ key_column_numbers[0]].equals( key);