manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1444015 - in /manifoldcf/branches/release-1.1-branch: ./ framework/core/src/main/java/org/apache/manifoldcf/core/database/ framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ framework/core/src/main/java/org/apache/manifoldcf...
Date Fri, 08 Feb 2013 13:59:17 GMT
Author: kwright
Date: Fri Feb  8 13:59:16 2013
New Revision: 1444015

URL: http://svn.apache.org/r1444015
Log:
Pull up fix for CONNECTORS-638 from trunk.

Modified:
    manifoldcf/branches/release-1.1-branch/   (props changed)
    manifoldcf/branches/release-1.1-branch/CHANGES.txt
    manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/ConnectionFactory.java
    manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPool.java
    manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPoolManager.java
    manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/WrappedConnection.java
    manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
    manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml
    manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml

Propchange: manifoldcf/branches/release-1.1-branch/
------------------------------------------------------------------------------
  Merged /manifoldcf/trunk:r1442101,1442171,1442313,1442578,1443521,1443655

Modified: manifoldcf/branches/release-1.1-branch/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.1-branch/CHANGES.txt?rev=1444015&r1=1444014&r2=1444015&view=diff
==============================================================================
--- manifoldcf/branches/release-1.1-branch/CHANGES.txt (original)
+++ manifoldcf/branches/release-1.1-branch/CHANGES.txt Fri Feb  8 13:59:16 2013
@@ -1,6 +1,12 @@
 ManifoldCF Change Log
 $Id$
 
+======================= Release 1.1.1 =====================
+
+CONNECTORS-638: Fix issue with database pool constriction due to
+connection expiration.
+(Erlend Garåsen, Maciej Li¿ewski, Karl Wright)
+
 ======================= Release 1.1 =====================
 
 CONNECTORS-630: Work around SolrJ paramname encoding bug.

Modified: manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/ConnectionFactory.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/ConnectionFactory.java?rev=1444015&r1=1444014&r2=1444015&view=diff
==============================================================================
--- manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/ConnectionFactory.java
(original)
+++ manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/ConnectionFactory.java
Fri Feb  8 13:59:16 2013
@@ -179,6 +179,7 @@ public class ConnectionFactory
     }
 
     public ConnectionPoolManager createPoolManager()
+      throws ManifoldCFException
     {
       synchronized (poolExistenceLock)
       {

Modified: manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPool.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPool.java?rev=1444015&r1=1444014&r2=1444015&view=diff
==============================================================================
--- manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPool.java
(original)
+++ manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPool.java
Fri Feb  8 13:59:16 2013
@@ -30,18 +30,22 @@ public class ConnectionPool
 {
   public static final String _rcsid = "@(#)$Id$";
 
-  protected String dbURL;
-  protected String userName;
-  protected String password;
+  protected final String dbURL;
+  protected final String userName;
+  protected final String password;
   protected volatile int freePointer;
   protected volatile int activeConnections;
   protected volatile boolean closed;
-  protected Connection[] freeConnections;
-  protected long[] connectionCleanupTimeouts;
-  protected long expiration;
+  protected final Connection[] freeConnections;
+  protected final long[] connectionCleanupTimeouts;
+  protected final long expiration;
+  
+  protected final boolean debug;
+  
+  protected final Set<WrappedConnection> outstandingConnections = new HashSet<WrappedConnection>();
   
   /** Constructor */
-  public ConnectionPool(String dbURL, String userName, String password, int maxConnections,
long expiration)
+  public ConnectionPool(String dbURL, String userName, String password, int maxConnections,
long expiration, boolean debug)
   {
     this.dbURL = dbURL;
     this.userName = userName;
@@ -52,6 +56,7 @@ public class ConnectionPool
     this.activeConnections = 0;
     this.closed = false;
     this.expiration = expiration;
+    this.debug = debug;
   }
   
   /** Obtain a connection from the pool.
@@ -62,6 +67,12 @@ public class ConnectionPool
   public WrappedConnection getConnection()
     throws SQLException, InterruptedException
   {
+    Exception instantiationException;
+    if (debug)
+      instantiationException = new Exception("Possibly leaked db connection");
+    else
+      instantiationException = null;
+    Connection rval = null;
     while (true)
     {
       synchronized (this)
@@ -70,12 +81,24 @@ public class ConnectionPool
         {
           if (closed)
             throw new InterruptedException("Pool already closed");
-          Connection rval = freeConnections[--freePointer];
+          rval = freeConnections[--freePointer];
           freeConnections[freePointer] = null;
-          return new WrappedConnection(this,rval);
+          break;
         }
         if (activeConnections == freeConnections.length)
         {
+          // If properly configured, we really shouldn't be getting here.
+          if (debug)
+          {
+            synchronized (outstandingConnections)
+            {
+              Logging.db.warn("Out of db connections, list of outstanding ones follows.");
+              for (WrappedConnection c : outstandingConnections)
+              {
+                Logging.db.warn("Found a possibly leaked db connection",c.getInstantiationException());
+              }
+            }
+          }
           // Wait until kicked; we hope something will free up...
           this.wait();
           continue;
@@ -85,22 +108,58 @@ public class ConnectionPool
         break;
       }
     }
-    
-    // Create a new connection.  If we fail at this we need to restore the number of active
connections, so catch any failures
-    Connection rval2 = null;
+
+    boolean returnedValue = true;
     try
     {
-      if (userName != null)
-        rval2 = DriverManager.getConnection(dbURL, userName, password);
-      else
-        rval2 = DriverManager.getConnection(dbURL);
+      if (rval == null)
+      {
+        if (userName != null)
+          rval = DriverManager.getConnection(dbURL, userName, password);
+        else
+          rval = DriverManager.getConnection(dbURL);
+      }
+
+      WrappedConnection wc = new WrappedConnection(this,rval,instantiationException);
+      if (debug)
+      {
+        synchronized (outstandingConnections)
+        {
+          outstandingConnections.add(wc);
+        }
+      }
+      return wc;
+    }
+    catch (Error e)
+    {
+      returnedValue = false;
+      throw e;
+    }
+    catch (RuntimeException e)
+    {
+      returnedValue = false;
+      throw e;
+    }
+    catch (SQLException e)
+    {
+      returnedValue = false;
+      throw e;
     }
     finally
     {
-      if (rval2 == null)
-        activeConnections--;
+      if (!returnedValue)
+      {
+        // We didn't finish.  Restore the pool to the correct form.
+        // Note: We should always be able to just return any current connection to the pool.
 This is
+        // safe because we reserved a slot when we decided to create the connection (if that's
what
+        // we did), or we just used a connection that was already allocated.  Either way,
we can put
+        // it into the pool.
+        if (rval != null)
+        {
+          release(rval);
+        }
+      }
     }
-    return new WrappedConnection(this,rval2);
   }
   
   /** Close down the pool.
@@ -133,15 +192,10 @@ public class ConnectionPool
     {
       if (connectionCleanupTimeouts[i] <= currentTime)
       {
-        try
-        {
-          freeConnections[i].close();
-        }
-        catch (SQLException e)
-        {
-          Logging.db.warn("Error closing pooled connection: "+e.getMessage(),e);
-        }
+        Connection c = freeConnections[i];
+        freeConnections[i] = null;
         freePointer--;
+        activeConnections--;
         if (freePointer == i)
         {
           freeConnections[i] = null;
@@ -152,18 +206,44 @@ public class ConnectionPool
           connectionCleanupTimeouts[i] = connectionCleanupTimeouts[freePointer];
           freeConnections[freePointer] = null;
         }
+        try
+        {
+         c.close();
+        }
+        catch (SQLException e)
+        {
+          Logging.db.warn("Error closing pooled connection: "+e.getMessage(),e);
+        }
       }
       else
         i++;
     }
   }
   
-  public synchronized void releaseConnection(Connection connection)
+  public void releaseConnection(WrappedConnection connection)
   {
-    freeConnections[freePointer] = connection;
-    connectionCleanupTimeouts[freePointer] = System.currentTimeMillis() + expiration;
-    freePointer++;
-    notifyAll();
+
+    if (debug)
+    {
+      synchronized (outstandingConnections)
+      {
+        outstandingConnections.remove(connection);
+      }
+    }
+
+    release(connection.getConnection());
+  }
+  
+  protected void release(Connection c)
+  {
+    synchronized (this)
+    {
+      freeConnections[freePointer] = c;
+      connectionCleanupTimeouts[freePointer] = System.currentTimeMillis() + expiration;
+      freePointer++;
+      notifyAll();
+    }
+    
   }
   
 }

Modified: manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPoolManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPoolManager.java?rev=1444015&r1=1444014&r2=1444015&view=diff
==============================================================================
--- manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPoolManager.java
(original)
+++ manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/ConnectionPoolManager.java
Fri Feb  8 13:59:16 2013
@@ -23,18 +23,24 @@ import javax.sql.*;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.core.system.ManifoldCF;
+
 /** An instance of this class manages a number of (independent) connection pools.
 */
 public class ConnectionPoolManager
 {
   public static final String _rcsid = "@(#)$Id$";
 
-  protected Map<String,ConnectionPool> poolMap;
-  protected ConnectionCloserThread connectionCloserThread;
+  protected final Map<String,ConnectionPool> poolMap;
+  protected final ConnectionCloserThread connectionCloserThread;
   protected volatile AtomicBoolean shuttingDown = new AtomicBoolean(false);
-
+  protected final boolean debug;
+  
   public ConnectionPoolManager(int count)
+    throws ManifoldCFException
   {
+    debug = ManifoldCF.getBooleanProperty(ManifoldCF.databaseConnectionTrackingProperty,
false);
     poolMap = new HashMap<String,ConnectionPool>(count);
     connectionCloserThread = new ConnectionCloserThread();
     connectionCloserThread.start();
@@ -54,7 +60,7 @@ public class ConnectionPoolManager
     throws ClassNotFoundException, InstantiationException, IllegalAccessException
   {
     Class.forName(driverClassName).newInstance();
-    ConnectionPool cp = new ConnectionPool(dbURL,userName,password,maxSize,expiration);
+    ConnectionPool cp = new ConnectionPool(dbURL,userName,password,maxSize,expiration,debug);
     poolMap.put(poolKey,cp);
     return cp;
   }

Modified: manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/WrappedConnection.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/WrappedConnection.java?rev=1444015&r1=1444014&r2=1444015&view=diff
==============================================================================
--- manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/WrappedConnection.java
(original)
+++ manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/jdbcpool/WrappedConnection.java
Fri Feb  8 13:59:16 2013
@@ -19,6 +19,7 @@
 package org.apache.manifoldcf.core.jdbcpool;
 
 import java.sql.*;
+import org.apache.manifoldcf.core.system.Logging;
 
 /** The class that represents a connection from a pool.
 */
@@ -28,12 +29,21 @@ public class WrappedConnection
 
   protected Connection connection;
   protected ConnectionPool owner;
+  /** Exception, to keep track of where the connection was allocated */
+  protected Exception instantiationException;
   
   /** Constructor */
   public WrappedConnection(ConnectionPool owner, Connection connection)
   {
+    this(owner,connection,null);
+  }
+  
+  /** Constructor */
+  public WrappedConnection(ConnectionPool owner, Connection connection, Exception instantiationException)
+  {
     this.owner = owner;
     this.connection = connection;
+    this.instantiationException = instantiationException;
   }
   
   /** Get the JDBC connection object.
@@ -47,9 +57,17 @@ public class WrappedConnection
   */
   public void release()
   {
-    owner.releaseConnection(this.connection);
+    owner.releaseConnection(this);
     this.connection = null;
   }
+  
+  /** Get instantiation exception.
+  */
+  public Exception getInstantiationException()
+  {
+    return instantiationException;
+  }
+  
 }
 
 

Modified: manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java?rev=1444015&r1=1444014&r2=1444015&view=diff
==============================================================================
--- manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
(original)
+++ manifoldcf/branches/release-1.1-branch/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
Fri Feb  8 13:59:16 2013
@@ -109,6 +109,8 @@ public class ManifoldCF
   public static final String databaseHandleMaxcountProperty = "org.apache.manifoldcf.database.maxhandles";
   /** Database handle timeout property */
   public static final String databaseHandleTimeoutProperty = "org.apache.manifoldcf.database.handletimeout";
+  /** Connection tracking debug property */
+  public static final String databaseConnectionTrackingProperty = "org.apache.manifoldcf.database.connectiontracking";
 
   // Database performance monitoring properties
   /** Elapsed time a query can take before a warning is output to the log, in seconds */

Modified: manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml?rev=1444015&r1=1444014&r2=1444015&view=diff
==============================================================================
--- manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml
(original)
+++ manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml
Fri Feb  8 13:59:16 2013
@@ -801,6 +801,7 @@ cd example
             <tr><td>org.apache.manifoldcf.synchdirectory</td><td>Yes,
if file-based synchronization class is used</td><td>Specifies the path of a synchronization
directory.  All ManifoldCF process owners <strong>must</strong> have read/write
privileges to this directory.</td></tr>
             <tr><td>org.apache.manifoldcf.database.maxhandles</td><td>No</td><td>Specifies
the maximum number of database connection handles that will by pooled.  Recommended value
is 200.</td></tr>
             <tr><td>org.apache.manifoldcf.database.handletimeout</td><td>No</td><td>Specifies
the maximum time a handle is to live before it is presumed dead.  Recommend a value of 604800,
which is the maximum allowable.</td></tr>
+            <tr><td>org.apache.manifoldcf.database.connectiontracking</td><td>No</td><td>True
or false.  When "true", will track all allocated database connection handles, and will dump
an allocation stack trace when the pool is exhausted.  Useful for diagnosing connection leaks.</td></tr>
             <tr><td>org.apache.manifoldcf.logconfigfile</td><td>No</td><td>Specifies
location of logging configuration file.</td></tr>
             <tr><td>org.apache.manifoldcf.database.name</td><td>No</td><td>Describes
database name for ManifoldCF; defaults to "dbname" if not specified.</td></tr>
             <tr><td>org.apache.manifoldcf.database.username</td><td>No</td><td>Describes
database user name for ManifoldCF; defaults to "manifoldcf" if not specified.</td></tr>

Modified: manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml?rev=1444015&r1=1444014&r2=1444015&view=diff
==============================================================================
--- manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml
(original)
+++ manifoldcf/branches/release-1.1-branch/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml
Fri Feb  8 13:59:16 2013
@@ -801,6 +801,7 @@ cd example
             <tr><td>org.apache.manifoldcf.synchdirectory</td><td>Yes,
if file-based synchronization class is used</td><td>Specifies the path of a synchronization
directory.  All ManifoldCF process owners <strong>must</strong> have read/write
privileges to this directory.</td></tr>
             <tr><td>org.apache.manifoldcf.database.maxhandles</td><td>No</td><td>Specifies
the maximum number of database connection handles that will by pooled.  Recommended value
is 200.</td></tr>
             <tr><td>org.apache.manifoldcf.database.handletimeout</td><td>No</td><td>Specifies
the maximum time a handle is to live before it is presumed dead.  Recommend a value of 604800,
which is the maximum allowable.</td></tr>
+            <tr><td>org.apache.manifoldcf.database.connectiontracking</td><td>No</td><td>True
or false.  When "true", will track all allocated database connection handles, and will dump
an allocation stack trace when the pool is exhausted.  Useful for diagnosing connection leaks.</td></tr>
             <tr><td>org.apache.manifoldcf.logconfigfile</td><td>No</td><td>Specifies
location of logging configuration file.</td></tr>
             <tr><td>org.apache.manifoldcf.database.name</td><td>No</td><td>Describes
database name for ManifoldCF; defaults to "dbname" if not specified.</td></tr>
             <tr><td>org.apache.manifoldcf.database.username</td><td>No</td><td>Describes
database user name for ManifoldCF; defaults to "manifoldcf" if not specified.</td></tr>



Mime
View raw message