manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1547801 - in /manifoldcf/branches/CONNECTORS-781/framework: agents/src/main/java/org/apache/manifoldcf/agents/system/ pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/
Date Wed, 04 Dec 2013 14:08:50 GMT
Author: kwright
Date: Wed Dec  4 14:08:50 2013
New Revision: 1547801

URL: http://svn.apache.org/r1547801
Log:
Add agents-level idle cleanup thread

Added:
    manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java
  (with props)
Modified:
    manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/AgentsDaemon.java
    manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java

Modified: manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/AgentsDaemon.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/AgentsDaemon.java?rev=1547801&r1=1547800&r2=1547801&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/AgentsDaemon.java
(original)
+++ manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/AgentsDaemon.java
Wed Dec  4 14:08:50 2013
@@ -36,6 +36,9 @@ public class AgentsDaemon
   * also takes on process cleanup where necessary. */
   protected AgentsThread agentsThread = null;
 
+  /** The idle cleanup thread. */
+  protected IdleCleanupThread idleCleanupThread = null;
+  
   /** Process ID for this agents daemon. */
   protected final String processID;
   
@@ -124,8 +127,11 @@ public class AgentsDaemon
   public void startAgents(IThreadContext threadContext)
     throws ManifoldCFException
   {
-    // Create and start agents thread.
+    // Create idle cleanup thread.
+    idleCleanupThread = new IdleCleanupThread(processID);
     agentsThread = new AgentsThread();
+    // Create and start agents thread.
+    idleCleanupThread.start();
     agentsThread.start();
   }
   
@@ -135,11 +141,17 @@ public class AgentsDaemon
     throws ManifoldCFException
   {
     // Shut down agents background thread.
-    while (agentsThread != null)
+    while (agentsThread != null || idleCleanupThread != null)
     {
-      agentsThread.interrupt();
-      if (!agentsThread.isAlive())
+      if (agentsThread != null)
+        agentsThread.interrupt();
+      if (idleCleanupThread != null)
+        idleCleanupThread.interrupt();
+      
+      if (agentsThread != null && !agentsThread.isAlive())
         agentsThread = null;
+      if (idleCleanupThread != null && !idleCleanupThread.isAlive())
+        idleCleanupThread = null;
     }
     
     // Shut down running agents services directly.

Added: manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java?rev=1547801&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java
(added)
+++ manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java
Wed Dec  4 14:08:50 2013
@@ -0,0 +1,130 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.agents.system;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+import java.util.*;
+
+/** This thread periodically calls the cleanup method in all connected output connectors.
 The ostensible purpose
+* is to allow the connectors to shutdown idle connections etc.
+*/
+public class IdleCleanupThread extends Thread
+{
+  public static final String _rcsid = "@(#)$Id$";
+
+  // Local data
+  /** Process ID */
+  protected final String processID;
+
+  /** Constructor.
+  */
+  public IdleCleanupThread(String processID)
+    throws ManifoldCFException
+  {
+    super();
+    this.processID = processID;
+    setName("Idle cleanup thread");
+    setDaemon(true);
+  }
+
+  public void run()
+  {
+    Logging.agents.debug("Start up idle cleanup thread");
+    try
+    {
+      // Create a thread context object.
+      IThreadContext threadContext = ThreadContextFactory.make();
+      // Get the cache handle.
+      ICacheManager cacheManager = CacheManagerFactory.make(threadContext);
+      // Get the output connector pool handle
+      IOutputConnectorPool outputConnectorPool = OutputConnectorPoolFactory.make(threadContext);
+      
+      // Loop
+      while (true)
+      {
+        // Do another try/catch around everything in the loop
+        try
+        {
+          // Do the cleanup
+          outputConnectorPool.pollAllConnectors();
+          cacheManager.expireObjects(System.currentTimeMillis());
+          
+          // Sleep for the retry interval.
+          ManifoldCF.sleep(15000L);
+        }
+        catch (ManifoldCFException e)
+        {
+          if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+            break;
+
+          if (e.getErrorCode() == ManifoldCFException.DATABASE_CONNECTION_ERROR)
+          {
+            Logging.agents.error("Idle cleanup thread aborting and restarting due to database
connection reset: "+e.getMessage(),e);
+            try
+            {
+              // Give the database a chance to catch up/wake up
+              ManifoldCF.sleep(10000L);
+            }
+            catch (InterruptedException se)
+            {
+              break;
+            }
+            continue;
+          }
+
+          // Log it, but keep the thread alive
+          Logging.agents.error("Exception tossed: "+e.getMessage(),e);
+
+          if (e.getErrorCode() == ManifoldCFException.SETUP_ERROR)
+          {
+            // Shut the whole system down!
+            System.exit(1);
+          }
+
+        }
+        catch (InterruptedException e)
+        {
+          // We're supposed to quit
+          break;
+        }
+        catch (OutOfMemoryError e)
+        {
+          System.err.println("agents process ran out of memory - shutting down");
+          e.printStackTrace(System.err);
+          System.exit(-200);
+        }
+        catch (Throwable e)
+        {
+          // A more severe error - but stay alive
+          Logging.agents.fatal("Error tossed: "+e.getMessage(),e);
+        }
+      }
+    }
+    catch (Throwable e)
+    {
+      // Severe error on initialization
+      System.err.println("agents process could not start - shutting down");
+      Logging.agents.fatal("IdleCleanupThread initialization error tossed: "+e.getMessage(),e);
+      System.exit(-300);
+    }
+
+  }
+
+}

Propchange: manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java
------------------------------------------------------------------------------
    svn:keywords = Id

Modified: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java?rev=1547801&r1=1547800&r2=1547801&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java
(original)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java
Wed Dec  4 14:08:50 2013
@@ -56,8 +56,6 @@ public class IdleCleanupThread extends T
       IThreadContext threadContext = ThreadContextFactory.make();
       // Get the cache handle.
       ICacheManager cacheManager = CacheManagerFactory.make(threadContext);
-      // Get the output connector pool handle
-      IOutputConnectorPool outputConnectorPool = OutputConnectorPoolFactory.make(threadContext);
       
       // Loop
       while (true)
@@ -67,7 +65,6 @@ public class IdleCleanupThread extends T
         {
           // Do the cleanup
           RepositoryConnectorFactory.pollAllConnectors(threadContext);
-          outputConnectorPool.pollAllConnectors();
           cacheManager.expireObjects(System.currentTimeMillis());
           
           // Sleep for the retry interval.



Mime
View raw message