db-derby-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d..@apache.org
Subject svn commit: r662446 - in /db/derby/code/trunk/java: engine/org/apache/derby/impl/sql/catalog/ engine/org/apache/derby/impl/sql/execute/ testing/org/apache/derbyTesting/functionTests/tests/lang/ testing/org/apache/derbyTesting/junit/
Date Mon, 02 Jun 2008 14:17:57 GMT
Author: dag
Date: Mon Jun  2 07:17:57 2008
New Revision: 662446

URL: http://svn.apache.org/viewvc?rev=662446&view=rev
Log:
DERBY-48 A connection request that has a default schema that is being created by another transaction
will fail to connect

Patch derby-48-7, which auto-creates the schema in a nested transaction if possible, thus
allowing early release of write locks used to auto-create the schema.

This has the side-effect of making auto-created schema persist even if
the user transaction that triggered it rolls back. Added a new test.


Added:
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/LazyDefaultSchemaCreationTest.java
  (with props)
Modified:
    db/derby/code/trunk/java/engine/org/apache/derby/impl/sql/catalog/DataDictionaryImpl.java
    db/derby/code/trunk/java/engine/org/apache/derby/impl/sql/execute/CreateSchemaConstantAction.java
    db/derby/code/trunk/java/engine/org/apache/derby/impl/sql/execute/DDLConstantAction.java
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/_Suite.java
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/junit/BaseTestCase.java

Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/sql/catalog/DataDictionaryImpl.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/sql/catalog/DataDictionaryImpl.java?rev=662446&r1=662445&r2=662446&view=diff
==============================================================================
--- db/derby/code/trunk/java/engine/org/apache/derby/impl/sql/catalog/DataDictionaryImpl.java
(original)
+++ db/derby/code/trunk/java/engine/org/apache/derby/impl/sql/catalog/DataDictionaryImpl.java
Mon Jun  2 07:17:57 2008
@@ -1522,7 +1522,8 @@
 						ti,
 						(TupleDescriptor) null,
 						(List) null,
-						false);
+						false,
+						tc);
 	}
 		
 	/**
@@ -1563,7 +1564,8 @@
 						ti,
 						(TupleDescriptor) null,
 						(List) null,
-						false);
+						false,
+						tc);
 	}
 
 
@@ -8042,8 +8044,8 @@
 	 * @param keyRow	The supplied ExecIndexRow for search
 	 * @param ti		The TabInfoImpl to use
 	 * @param parentTupleDescriptor		The parentDescriptor, if applicable.
-	 * @param list		The list to build, if supplied.  If null, then caller expects
-	 *					a single descriptor
+	 * @param list      The list to build, if supplied.  If null, then
+	 *                  caller expects a single descriptor
 	 * @param forUpdate	Whether or not to open the index for update.
 	 *
 	 * @return	The last matching descriptor
@@ -8060,6 +8062,76 @@
 						boolean forUpdate)
 			throws StandardException
 	{
+		// Get the current transaction controller
+		TransactionController tc = getTransactionCompile();
+
+		return getDescriptorViaIndexMinion(indexId,
+										   keyRow,
+										   scanQualifiers,
+										   ti,
+										   parentTupleDescriptor,
+										   list,
+										   forUpdate,
+										   tc);
+	}
+
+	/**
+	 * Return a (single or list of) catalog row descriptor(s) from a
+	 * system table where the access is from the index to the heap.
+	 *
+	 * This overload variant takes an explicit tc, in contrast to the normal
+	 * one which uses the one returned by getTransactionCompile.
+	 *
+	 * @param indexId	The id of the index (0 to # of indexes on table) to use
+	 * @param keyRow	The supplied ExecIndexRow for search
+	 * @param ti		The TabInfoImpl to use
+	 * @param parentTupleDescriptor		The parentDescriptor, if applicable.
+	 * @param list      The list to build, if supplied.  If null, then
+	 *					caller expects a single descriptor
+	 * @param forUpdate	Whether or not to open the index for update.
+	 * @param tc        Transaction controller
+	 *
+	 * @return	The last matching descriptor
+	 *
+	 * @exception StandardException		Thrown on error
+	 */
+	private final TupleDescriptor getDescriptorViaIndex(
+						int indexId,
+						ExecIndexRow keyRow,
+						ScanQualifier [][] scanQualifiers,
+						TabInfoImpl ti,
+						TupleDescriptor parentTupleDescriptor,
+						List list,
+						boolean forUpdate,
+						TransactionController tc)
+			throws StandardException
+	{
+		if (tc == null) {
+			tc = getTransactionCompile();
+		}
+
+		return getDescriptorViaIndexMinion(indexId,
+										   keyRow,
+										   scanQualifiers,
+										   ti,
+										   parentTupleDescriptor,
+										   list,
+										   forUpdate,
+										   tc);
+	}
+
+
+	private final TupleDescriptor getDescriptorViaIndexMinion(
+						int indexId,
+						ExecIndexRow keyRow,
+						ScanQualifier [][] scanQualifiers,
+						TabInfoImpl ti,
+						TupleDescriptor parentTupleDescriptor,
+						List list,
+						boolean forUpdate,
+						TransactionController tc)
+			throws StandardException
+	{
 		CatalogRowFactory		rf = ti.getCatalogRowFactory();
 		ConglomerateController	heapCC;
 		ExecIndexRow	  		indexRow1;
@@ -8067,12 +8139,8 @@
 		ExecRow 				outRow;
 		RowLocation				baseRowLocation;
 		ScanController			scanController;
-		TransactionController	tc;
 		TupleDescriptor			td = null;
 
-		// Get the current transaction controller
-		tc = getTransactionCompile();
-
 		outRow = rf.makeEmptyRow();
 
 		heapCC = tc.openConglomerate(

Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/sql/execute/CreateSchemaConstantAction.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/sql/execute/CreateSchemaConstantAction.java?rev=662446&r1=662445&r2=662446&view=diff
==============================================================================
--- db/derby/code/trunk/java/engine/org/apache/derby/impl/sql/execute/CreateSchemaConstantAction.java
(original)
+++ db/derby/code/trunk/java/engine/org/apache/derby/impl/sql/execute/CreateSchemaConstantAction.java
Mon Jun  2 07:17:57 2008
@@ -98,9 +98,35 @@
 	public void	executeConstantAction( Activation activation )
 						throws StandardException
 	{
+		TransactionController tc = activation.
+			getLanguageConnectionContext().getTransactionExecute();
+
+		executeConstantActionMinion(activation, tc);
+	}
+
+	/**
+	 *	This is the guts of the Execution-time logic for CREATE SCHEMA.
+	 *  This is variant is used when we to pass in a tc other than the default
+	 *  used in executeConstantAction(Activation).
+	 *
+	 * @param activation current activation
+	 * @param tc transaction controller
+	 *
+	 * @exception StandardException		Thrown on failure
+	 */
+	public void	executeConstantAction(Activation activation,
+									  TransactionController tc)
+			throws StandardException {
+
+		executeConstantActionMinion(activation, tc);
+	}
+
+	private void executeConstantActionMinion(Activation activation,
+											 TransactionController tc)
+			throws StandardException {
+
 		LanguageConnectionContext lcc = activation.getLanguageConnectionContext();
 		DataDictionary dd = lcc.getDataDictionary();
-		TransactionController tc = lcc.getTransactionExecute();
 		DataDescriptorGenerator ddg = dd.getDataDescriptorGenerator();
 
 		SchemaDescriptor sd = dd.getSchemaDescriptor(schemaName, lcc.getTransactionExecute(), false);

Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/sql/execute/DDLConstantAction.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/sql/execute/DDLConstantAction.java?rev=662446&r1=662445&r2=662446&view=diff
==============================================================================
--- db/derby/code/trunk/java/engine/org/apache/derby/impl/sql/execute/DDLConstantAction.java
(original)
+++ db/derby/code/trunk/java/engine/org/apache/derby/impl/sql/execute/DDLConstantAction.java
Mon Jun  2 07:17:57 2008
@@ -27,6 +27,9 @@
 import org.apache.derby.catalog.UUID;
 import org.apache.derby.iapi.error.StandardException;
 import org.apache.derby.iapi.reference.SQLState;
+import org.apache.derby.iapi.reference.Property;
+import org.apache.derby.iapi.services.property.PropertyUtil;
+import org.apache.derby.iapi.services.sanity.SanityManager;
 import org.apache.derby.iapi.sql.Activation;
 import org.apache.derby.iapi.sql.conn.Authorizer;
 import org.apache.derby.iapi.sql.conn.LanguageConnectionContext;
@@ -79,8 +82,8 @@
 	   the passed in schema.
 	 *
 	 * @param dd the data dictionary
-	   @param activation activation
-	   @param schemaName name of the schema
+	 * @param activation activation
+	 * @param schemaName name of the schema
 	 *
 	 * @return the schema descriptor
 	 *
@@ -92,31 +95,98 @@
 						String schemaName)
 		throws StandardException
 	{
-		TransactionController tc = activation.getLanguageConnectionContext().getTransactionExecute();
+		TransactionController tc = activation.
+			getLanguageConnectionContext().getTransactionExecute();
+
 		SchemaDescriptor sd = dd.getSchemaDescriptor(schemaName, tc, false);
 
 		if (sd == null || sd.getUUID() == null) {
-            ConstantAction csca 
+            CreateSchemaConstantAction csca
                 = new CreateSchemaConstantAction(schemaName, (String) null);
 
-            try {
-                csca.executeConstantAction(activation);
-            } catch (StandardException se) {
-                if (se.getMessageId()
-                    .equals(SQLState.LANG_OBJECT_ALREADY_EXISTS)) {
-                    // Ignore "Schema already exists". Another thread has 
-                    // probably created it after we checked for it
-                } else {
-                    throw se;
-                }
-            }
-            
+			// DERBY-48: This operation creates a schema and we don't
+			// want to hold a lock for SYSSCHEMAS for the duration of
+			// the user transaction, so we perform the creation in a
+			// nested transaction if possible.
+			TransactionController useTc    = null;
+			TransactionController nestedTc = null;
+
+			try {
+				nestedTc = tc.startNestedUserTransaction(false);
+				useTc = nestedTc;
+			} catch (StandardException e) {
+				if (SanityManager.DEBUG) {
+					SanityManager.THROWASSERT(
+						"Unexpected: not able to start nested transaction " +
+						"to auto-create schema", e);
+				}
+				useTc = tc;
+			}
+
+			// Try max twice: if nested transaction times out, try
+			// again in the outer transaction because it may be a
+			// self-lock, that is, the outer transaction may hold some
+			// lock(s) that make the nested transaction attempt to set
+			// a write lock time out.  Trying it again in the outer
+			// transaction will then succeed. If the reason is some
+			// other transaction barring us, trying again in the outer
+			// transaction will possibly time out again.
+			//
+			// Also, if creating a nested transaction failed, only try
+			// once in the outer transaction.
+			while (true) {
+				try {
+					csca.executeConstantAction(activation, useTc);
+				} catch (StandardException se) {
+					if (se.getMessageId().equals(SQLState.LOCK_TIMEOUT)) {
+						// We don't test for SQLState.DEADLOCK or
+						// .LOCK_TIMEOUT_LOG here because a) if it is a
+						// deadlock, it may be better to expose it, and b)
+						// LOCK_TIMEOUT_LOG happens when the app has set
+						// derby.locks.deadlockTrace=true, in which case we
+						// don't want to mask the timeout.  So in both the
+						// latter cases we just throw.
+						if (useTc == nestedTc) {
+
+							// clean up after use of nested transaction,
+							// then try again in outer transaction
+							useTc = tc;
+							nestedTc.destroy();
+							continue;
+						}
+					} else if (se.getMessageId()
+							.equals(SQLState.LANG_OBJECT_ALREADY_EXISTS)) {
+						// Ignore "Schema already exists". Another thread has
+						// probably created it after we checked for it
+						break;
+					}
+
+					// We got an non-expected exception, either in
+					// the nested transaction or in the outer
+					// transaction; we had better pass that on
+					if (useTc == nestedTc) {
+						nestedTc.destroy();
+					}
+
+					throw se;
+				}
+				break;
+			}
+
+			// We either succeeded or got LANG_OBJECT_ALREADY_EXISTS.
+			// Clean up if we did this in a nested transaction.
+			if (useTc == nestedTc) {
+				nestedTc.commit();
+				nestedTc.destroy();
+			}
+
 			sd = dd.getSchemaDescriptor(schemaName, tc, true);
 		}
 
 		return sd;
 	}
 
+
 	/**
 	 * Lock the table in exclusive or share mode to prevent deadlocks.
 	 *

Added: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/LazyDefaultSchemaCreationTest.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/LazyDefaultSchemaCreationTest.java?rev=662446&view=auto
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/LazyDefaultSchemaCreationTest.java
(added)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/LazyDefaultSchemaCreationTest.java
Mon Jun  2 07:17:57 2008
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.derbyTesting.functionTests.tests.lang;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.util.Properties;
+
+import junit.framework.Test;
+import junit.framework.TestSuite;
+
+import org.apache.derbyTesting.junit.JDBC;
+import org.apache.derbyTesting.junit.BaseJDBCTestCase;
+import org.apache.derbyTesting.junit.TestConfiguration;
+import org.apache.derbyTesting.junit.DatabasePropertyTestSetup;
+
+/**
+ * Tests the lazy creation functionality of default schema: the schema
+ * is only first created when the first database object is created in
+ * the schema.
+ */
+public class LazyDefaultSchemaCreationTest extends BaseJDBCTestCase {
+
+    final private static String LOCK_TIMEOUT = "40XL1";
+    final private static String LOCK_TIMEOUT_LOG = "40XL2";
+
+    /**
+     * Creates a new {@code LazyDefaultSchemaCreationTest} instance.
+     *
+     * @param name the name of the test
+     */
+    public LazyDefaultSchemaCreationTest(String name) {
+        super(name);
+    }
+
+
+    /**
+     * Reproduces hang seen in DERBY-48
+     */
+    public void testDerby48testNewSchemaHang () throws SQLException
+    {
+        Connection c1 = openUserConnection("newuser");
+        c1.setAutoCommit(false);
+        Statement s1 = c1.createStatement();
+
+        // Will auto-create schema NEWUSER:
+        s1.executeUpdate("create table t1(i int)");
+        s1.close();
+
+        // DERBY-48: The next connect causes a hang on write lock the
+        // new schema row being created by c1 that is not yet
+        // committed if the fix for DERBY-48 is not yet in place.
+        // The fix makes the the auto-create happen in a nested transaction
+        // which commit immediately, so the hang should not be present.
+
+        Connection c2 = null;
+
+        try {
+            c2 = openUserConnection("newuser");
+        } catch (SQLException e) {
+            if (e.getSQLState().equals(LOCK_TIMEOUT)) {
+                c1.rollback();
+                c1.close();
+                fail("DERBY-48 still seen", e);
+            } else {
+                throw e;
+            }
+        }
+
+        c1.rollback();
+
+        // Since the auto-create happened in a nested transaction
+        // which has committed, the schema should still be around
+        // after the rollback. Note that this is a side-effect of the
+        // fix for DERBY-48, not required behavior for SQL, but it is
+        // user visible behavior, so we test it here to make sure that
+        // patch works as intended:
+
+        JDBC.assertSingleValueResultSet(
+            c1.createStatement().executeQuery(
+                "select schemaname from sys.sysschemas " +
+                "where schemaname='NEWUSER'"),
+            "NEWUSER");
+
+        c1.rollback();
+
+        c1.close();
+        c2.close();
+    }
+
+    /**
+     * Test that we recover from self locking in the auto-create
+     * nested transaction (cf solution for DERBY-48).
+     */
+    public void testDerby48SelfLockingRecovery () throws SQLException
+    {
+        Connection c1 = openUserConnection("newuser");
+        c1.setAutoCommit(false);
+        c1.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE);
+        Statement s1 = c1.createStatement();
+
+        // Set read locks in parent transaction
+        s1.executeQuery("select count(*) from sys.sysschemas");
+
+        // ..which conflicts with the auto-create in a subtransaction
+        // which will self-lock here, but should recover to try again
+        // in outer transaction:
+        s1.executeUpdate("create table t1(i int)");
+
+        JDBC.assertSingleValueResultSet(
+            s1.executeQuery(
+                "select schemaname from sys.sysschemas " +
+                "where schemaname='NEWUSER'"),
+            "NEWUSER");
+
+        c1.rollback();
+
+        // Since the fallback does the auto-create of the schema in
+        // the outer transaction, a rollback will remove it:
+        JDBC.assertEmpty(
+            s1.executeQuery
+            ("select * from sys.sysschemas where schemaname='NEWUSER'"));
+
+        c1.rollback();
+    }
+
+    /**
+     * Test that we do get to see the self locking in the auto-create
+     * nested transaction (cf solution for DERBY-48) when deadlock
+     * detection is on, i.e. 40XL2 (LOCK_TIMEOUT_LOG) rather than
+     * 40XL1 (LOCK_TIMEOUT) happens.
+     */
+    public void testDerby48SelfLockingRecoveryDeadlockDetectionOn ()
+            throws SQLException
+    {
+        Connection c1 = openUserConnection("newuser");
+        c1.setAutoCommit(false);
+        c1.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE);
+        Statement s1 = c1.createStatement();
+
+        s1.executeUpdate(
+            "CALL SYSCS_UTIL.SYSCS_SET_DATABASE_PROPERTY(" +
+                "'derby.locks.deadlockTrace', 'true')");
+
+
+        // Set read locks in parent transaction
+        s1.executeQuery("select count(*) from sys.sysschemas");
+
+        // ..which conflicts with the auto-create in a subtransaction
+        // which will self-lock here, but should throw now:
+        // in outer transaction:
+        try {
+            s1.executeUpdate("create table t1(i int)");
+            fail("Expected exception " + LOCK_TIMEOUT_LOG);
+        } catch (SQLException e) {
+            assertSQLState("Expected state: ", LOCK_TIMEOUT_LOG, e);
+        }
+
+        JDBC.assertEmpty(
+            s1.executeQuery
+            ("select * from sys.sysschemas where schemaname='NEWUSER'"));
+
+        c1.rollback();
+    }
+
+    protected void  tearDown() throws Exception {
+        try {
+            createStatement().executeUpdate("drop schema newuser restrict");
+        } catch (SQLException e) {
+            // If not created by the fixture:
+            assertSQLState("Expected state: ", "42Y07", e);
+        }
+
+        super.tearDown();
+    }
+
+    public static Test suite() {
+        TestSuite suite = new TestSuite("LazyDefaultSchemaCreationTest");
+
+        TestSuite[] suites = {
+            new TestSuite("LazyDefaultSchemaCreationTest:embedded"),
+            new TestSuite("LazyDefaultSchemaCreationTest:clientServer") };
+
+        for (int i=0; i < 2; i++) {
+            suites[i].addTest(DatabasePropertyTestSetup.setLockTimeouts
+                          (new LazyDefaultSchemaCreationTest
+                           ("testDerby48testNewSchemaHang"),2,1));
+
+            suites[i].addTest(DatabasePropertyTestSetup.setLockTimeouts
+                          (new LazyDefaultSchemaCreationTest
+                           ("testDerby48SelfLockingRecovery"),2,1));
+
+            Properties p = new Properties();
+            p.setProperty("derby.locks.deadlockTrace", "true");
+
+            suites[i].addTest
+                (DatabasePropertyTestSetup.setLockTimeouts
+                 (new DatabasePropertyTestSetup
+                  (new LazyDefaultSchemaCreationTest
+                   ("testDerby48SelfLockingRecoveryDeadlockDetectionOn"),
+                   p, false),
+                  2,   // deadlock timeout
+                  1)); // wait timeout
+
+            if (i == 0) {
+                suite.addTest(suites[i]);
+            } else {
+                suite.addTest(
+                    TestConfiguration.clientServerDecorator(suites[i]));
+            }
+
+
+        }
+
+        return suite;
+    }
+}

Propchange: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/LazyDefaultSchemaCreationTest.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/_Suite.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/_Suite.java?rev=662446&r1=662445&r2=662446&view=diff
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/_Suite.java
(original)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/lang/_Suite.java
Mon Jun  2 07:17:57 2008
@@ -109,6 +109,7 @@
         suite.addTest(SimpleTest.suite());
         suite.addTest(GrantRevokeDDLTest.suite());
         suite.addTest(ReleaseCompileLocksTest.suite());
+        suite.addTest(LazyDefaultSchemaCreationTest.suite());
         suite.addTest(ErrorCodeTest.suite());
         suite.addTest(TimestampArithTest.suite());
         suite.addTest(SpillHashTest.suite());

Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/junit/BaseTestCase.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/junit/BaseTestCase.java?rev=662446&r1=662445&r2=662446&view=diff
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/junit/BaseTestCase.java (original)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/junit/BaseTestCase.java Mon Jun
 2 07:17:57 2008
@@ -21,6 +21,7 @@
 
 import junit.framework.Assert;
 import junit.framework.TestCase;
+import junit.framework.AssertionFailedError;
 
 import java.io.BufferedInputStream;
 import java.io.File;
@@ -528,4 +529,20 @@
     {
         DropDatabaseSetup.removeDirectory(dir);
     }
+
+    /**
+     * Fail; attaching an exception for more detail on cause.
+     *
+     * @param msg message explaining the failure
+     * @param e exception related to the cause
+     *
+     * @exception AssertionFailedError
+     */
+    public static void fail(String msg, Exception e)
+            throws AssertionFailedError {
+
+        AssertionFailedError ae = new AssertionFailedError(msg);
+        ae.initCause(e);
+        throw ae;
+    }
 } // End class BaseTestCase



Mime
View raw message