manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1547019 - in /manifoldcf/branches/CONNECTORS-781/framework: agents/src/main/java/org/apache/manifoldcf/agents/ agents/src/main/java/org/apache/manifoldcf/agents/system/ combined-service/src/main/java/org/apache/manifoldcf/combinedservice/ ...
Date Mon, 02 Dec 2013 13:45:49 GMT
Author: kwright
Date: Mon Dec  2 13:45:49 2013
New Revision: 1547019

URL: http://svn.apache.org/r1547019
Log:
Switch over to using the instantiable class for agent daemon management.

Modified:
    manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentRun.java
    manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentStop.java
    manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java
    manifoldcf/branches/CONNECTORS-781/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/ServletListener.java
    manifoldcf/branches/CONNECTORS-781/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java
    manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java

Modified: manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentRun.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentRun.java?rev=1547019&r1=1547018&r2=1547019&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentRun.java
(original)
+++ manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentRun.java
Mon Dec  2 13:45:49 2013
@@ -59,8 +59,9 @@ public class AgentRun extends BaseAgents
       
       Logging.root.info("Running...");
       // Register hook first so stopAgents() not required
-      ManifoldCF.registerAgentsShutdownHook(tc, processID);
-      ManifoldCF.runAgents(tc, processID);
+      AgentsDaemon ad = new AgentsDaemon(processID);
+      ad.registerAgentsShutdownHook(tc);
+      ad.runAgents(tc);
       Logging.root.info("Shutting down...");
     }
     catch (ManifoldCFException e)

Modified: manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentStop.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentStop.java?rev=1547019&r1=1547018&r2=1547019&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentStop.java
(original)
+++ manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentStop.java
Mon Dec  2 13:45:49 2013
@@ -37,7 +37,7 @@ public class AgentStop extends BaseAgent
     // As part of the work for CONNECTORS-781, this method is now synchronous.
     // We assert the shutdown signal, and then wait until all active services have shut down.
     ILockManager lockManager = LockManagerFactory.make(tc);
-    ManifoldCF.assertAgentsShutdownSignal(tc);
+    AgentsDaemon.assertAgentsShutdownSignal(tc);
     try
     {
       Logging.root.info("Shutdown signal sent");
@@ -61,7 +61,7 @@ public class AgentStop extends BaseAgent
     finally
     {
       // Clear shutdown signal
-      ManifoldCF.clearAgentsShutdownSignal(tc);
+      AgentsDaemon.clearAgentsShutdownSignal(tc);
     }
   }
 

Modified: manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java?rev=1547019&r1=1547018&r2=1547019&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java
(original)
+++ manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java
Mon Dec  2 13:45:49 2013
@@ -32,11 +32,6 @@ public class ManifoldCF extends org.apac
   // Agents initialized flag
   protected static boolean agentsInitialized = false;
   
-  /** This is the place we keep track of the agents we've started. */
-  protected static Map<String,IAgent> runningHash = new HashMap<String,IAgent>();
-  /** This flag prevents startAgents() from starting anything once stopAgents() has been
called. */
-  protected static boolean stopAgentsRun = false;
-  
   /** Initialize environment.
   */
   public static void initializeEnvironment(IThreadContext threadContext)
@@ -88,10 +83,6 @@ public class ManifoldCF extends org.apac
     synchronized (initializeFlagLock)
     {
       org.apache.manifoldcf.core.system.ManifoldCF.resetEnvironment(threadContext);
-      synchronized (runningHash)
-      {
-        stopAgentsRun = false;
-      }
     }
   }
 
@@ -128,317 +119,6 @@ public class ManifoldCF extends org.apac
     mgr.deinstall();
   }
 
-  // There are a number of different ways of running the agents framework.
-  // (1) Repeatedly call checkAgents(), and when all done make sure to call stopAgents().
-  // (2) Call registerAgentsShutdownHook(), then repeatedly run checkAgents(),  Agent shutdown
happens on JVM exit.
-  // (3) Call runAgents(), which will wait for someone else to call assertAgentsShutdownSignal().
 Before exit, stopAgents() must be called.
-  // (4) Call registerAgentsShutdownHook(), then call runAgents(), which will wait for someone
else to call assertAgentsShutdownSignal().  Shutdown happens on JVM exit.
-  
-  /** Assert shutdown signal.
-  */
-  public static void assertAgentsShutdownSignal(IThreadContext threadContext)
-    throws ManifoldCFException
-  {
-    ILockManager lockManager = LockManagerFactory.make(threadContext);
-    lockManager.setGlobalFlag(agentShutdownSignal);
-  }
-  
-  /** Clear shutdown signal.
-  */
-  public static void clearAgentsShutdownSignal(IThreadContext threadContext)
-    throws ManifoldCFException
-  {
-    ILockManager lockManager = LockManagerFactory.make(threadContext);
-    lockManager.clearGlobalFlag(agentShutdownSignal);
-  }
-
-
-  /** Register agents shutdown hook.
-  * Call this ONCE before calling startAgents or checkAgents the first time, if you want
automatic cleanup of agents on JVM stop.
-  */
-  public static void registerAgentsShutdownHook(IThreadContext threadContext, String processID)
-    throws ManifoldCFException
-  {
-    // Create the shutdown hook for agents.  All activity will be keyed off of runningHash,
so it is safe to do this under all conditions.
-    org.apache.manifoldcf.core.system.ManifoldCF.addShutdownHook(new AgentsShutdownHook(processID));
-  }
-  
-  /** Agent service name prefix (followed by agent class name) */
-  public static final String agentServicePrefix = "AGENT_";
-  
-  protected static AgentsThread agentsThread = null;
-
-  /** Run agents process.
-  * This method will not return until a shutdown signal is sent.
-  */
-  public static void runAgents(IThreadContext threadContext, String processID)
-    throws ManifoldCFException
-  {
-    ILockManager lockManager = LockManagerFactory.make(threadContext);
-
-    // Don't come up at all if shutdown signal in force
-    if (lockManager.checkGlobalFlag(agentShutdownSignal))
-      return;
-
-    // Create and start agents thread.
-    startAgents(threadContext, processID);
-    
-    while (true)
-    {
-      // Any shutdown signal yet?
-      if (lockManager.checkGlobalFlag(agentShutdownSignal))
-        break;
-          
-      try
-      {
-        ManifoldCF.sleep(5000L);
-      }
-      catch (InterruptedException e)
-      {
-        break;
-      }
-    }
-    
-  }
-
-  /** Start agents thread.
-  */
-  public static void startAgents(IThreadContext threadContext, String processID)
-    throws ManifoldCFException
-  {
-    // Create and start agents thread.
-    agentsThread = new AgentsThread(processID);
-    agentsThread.start();
-  }
-  
-  /** Stop all started agents.
-  */
-  public static void stopAgents(IThreadContext threadContext, String processID)
-    throws ManifoldCFException
-  {
-    // Shut down agents background thread.
-    while (agentsThread != null)
-    {
-      agentsThread.interrupt();
-      if (!agentsThread.isAlive())
-        agentsThread = null;
-    }
-    
-    // Shut down running agents services directly.
-    ILockManager lockManager = LockManagerFactory.make(threadContext);
-    synchronized (runningHash)
-    {
-      // This is supposedly safe; iterator remove is used
-      Iterator<String> iter = runningHash.keySet().iterator();
-      while (iter.hasNext())
-      {
-        String className = iter.next();
-        IAgent agent = runningHash.get(className);
-        // Stop it
-        agent.stopAgent(threadContext);
-        lockManager.endServiceActivity(getAgentsClassServiceType(className), processID);
-        iter.remove();
-        agent.cleanUp(threadContext);
-      }
-    }
-    // Done.
-  }
-
-  protected static String getAgentsClassServiceType(String agentClassName)
-  {
-    return agentServicePrefix + agentClassName;
-  }
-  
-  /** Agents thread.  This runs in background until interrupted, at which point
-  * it shuts down.  Its responsibilities include cleaning up after dead processes,
-  * as well as starting newly-registered agent processes, and terminating ones that disappear.
-  */
-  protected static class AgentsThread extends Thread
-  {
-    protected final String processID;
-    
-    public AgentsThread(String processID)
-    {
-      super();
-      this.processID = processID;
-      setName("Agents thread");
-      setDaemon(true);
-    }
-    
-    public void run()
-    {
-      try
-      {
-        IThreadContext threadContext = ThreadContextFactory.make();
-        while (true)
-        {
-          try
-          {
-            if (Thread.currentThread().isInterrupted())
-              throw new ManifoldCFException("Interrupted",ManifoldCFException.INTERRUPTED);
-
-            checkAgents(threadContext, processID);
-            ManifoldCF.sleep(5000L);
-          }
-          catch (InterruptedException e)
-          {
-            break;
-          }
-          catch (ManifoldCFException e)
-          {
-            if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
-              break;
-            Logging.agents.error("Exception tossed: "+e.getMessage(),e);
-          }
-          catch (OutOfMemoryError e)
-          {
-            System.err.println("Agents process ran out of memory - shutting down");
-            e.printStackTrace(System.err);
-            System.exit(-200);
-          }
-          catch (Throwable e)
-          {
-            Logging.agents.fatal("Error tossed: "+e.getMessage(),e);
-          }
-        }
-      }
-      catch (Throwable e)
-      {
-        // Severe error on initialization
-        System.err.println("Agents process could not start - shutting down");
-        Logging.agents.fatal("AgentThread initialization error tossed: "+e.getMessage(),e);
-        System.exit(-300);
-      }
-    }
-  }
-
-  /** Start all not-running agents.
-  *@param threadContext is the thread context.
-  */
-  protected static void checkAgents(IThreadContext threadContext, String processID)
-    throws ManifoldCFException
-  {
-    ILockManager lockManager = LockManagerFactory.make(threadContext);
-    // Get agent manager
-    IAgentManager manager = AgentManagerFactory.make(threadContext);
-    synchronized (runningHash)
-    {
-      String[] classes = manager.getAllAgents();
-      Set<String> currentAgentClasses = new HashSet<String>();
-
-      int i = 0;
-      while (i < classes.length)
-      {
-        String className = classes[i++];
-        if (runningHash.get(className) == null)
-        {
-          // Start this agent
-          IAgent agent = AgentFactory.make(className);
-          agent.initialize(threadContext);
-          try
-          {
-            // Throw a lock, so that cleanup processes and startup processes don't collide.
-            String serviceType = getAgentsClassServiceType(className);
-            lockManager.registerServiceBeginServiceActivity(serviceType, processID, new CleanupAgent(threadContext,
agent, processID));
-            // There is a potential race condition where the agent has been started but hasn't
yet appeared in runningHash.
-            // But having runningHash be the synchronizer for this activity will prevent
any problems.
-            agent.startAgent(threadContext, processID);
-            // Successful!
-            runningHash.put(className,agent);
-          }
-          catch (ManifoldCFException e)
-          {
-            if (e.getErrorCode() != ManifoldCFException.INTERRUPTED)
-              agent.cleanUp(threadContext);
-            throw e;
-          }
-        }
-        currentAgentClasses.add(className);
-      }
-
-      // Go through running hash and look for agents processes that have left
-      Iterator<String> runningAgentsIterator = runningHash.keySet().iterator();
-      while (runningAgentsIterator.hasNext())
-      {
-        String runningAgentClass = runningAgentsIterator.next();
-        if (!currentAgentClasses.contains(runningAgentClass))
-        {
-          // Shut down this one agent.
-          IAgent agent = runningHash.get(runningAgentClass);
-          // Stop it
-          agent.stopAgent(threadContext);
-          lockManager.endServiceActivity(getAgentsClassServiceType(runningAgentClass), processID);
-          runningAgentsIterator.remove();
-          agent.cleanUp(threadContext);
-        }
-      }
-    }
-
-    synchronized (runningHash)
-    {
-      // For every class we're supposed to be running, find registered but no-longer-active
instances and clean
-      // up after them.
-      for (String agentsClass : runningHash.keySet())
-      {
-        IAgent agent = runningHash.get(agentsClass);
-        IServiceCleanup cleanup = new CleanupAgent(threadContext, agent, processID);
-        String agentsClassServiceType = getAgentsClassServiceType(agentsClass);
-        while (!lockManager.cleanupInactiveService(agentsClassServiceType, cleanup))
-        {
-          // Loop until no more inactive services
-        }
-      }
-    }
-    
-  }
-
-  
-  protected static class CleanupAgent implements IServiceCleanup
-  {
-    protected final IAgent agent;
-    protected final IThreadContext threadContext;
-    protected final String processID;
-
-    public CleanupAgent(IThreadContext threadContext, IAgent agent, String processID)
-    {
-      this.agent = agent;
-      this.threadContext = threadContext;
-      this.processID = processID;
-    }
-    
-    /** Clean up after the specified service.  This method will block any startup of the
specified
-    * service for as long as it runs.
-    *@param serviceName is the name of the service.
-    */
-    @Override
-    public void cleanUpService(String serviceName)
-      throws ManifoldCFException
-    {
-      agent.cleanUpAgentData(threadContext, processID, serviceName);
-    }
-
-    /** Clean up after ALL services of the type on the cluster.
-    */
-    @Override
-    public void cleanUpAllServices()
-      throws ManifoldCFException
-    {
-      agent.cleanUpAllAgentData(threadContext, processID);
-    }
-    
-    /** Perform cluster initialization - that is, whatever is needed presuming that the
-    * cluster has been down for an indeterminate period of time, but is otherwise in a clean
-    * state.
-    */
-    @Override
-    public void clusterInit()
-      throws ManifoldCFException
-    {
-      agent.clusterInit(threadContext);
-    }
-
-  }
-  
   /** Signal output connection needs redoing.
   * This is called when something external changed on an output connection, and
   * therefore all associated documents must be reindexed.
@@ -456,30 +136,6 @@ public class ManifoldCF extends org.apac
     AgentManagerFactory.noteOutputConnectionChange(threadContext,connectionName);
   }
   
-  /** Agents shutdown hook class */
-  protected static class AgentsShutdownHook implements IShutdownHook
-  {
-    protected final String processID;
-
-    public AgentsShutdownHook(String processID)
-    {
-      this.processID = processID;
-    }
-    
-    public void doCleanup()
-      throws ManifoldCFException
-    {
-      // Shutting down in this way must prevent startup from taking place.
-      synchronized (runningHash)
-      {
-        stopAgentsRun = true;
-      }
-      IThreadContext tc = ThreadContextFactory.make();
-      stopAgents(tc,processID);
-    }
-    
-  }
-  
   // Helper methods for API support.  These are made public so connectors can use them to
implement the executeCommand method.
   
   // These are the universal node types.

Modified: manifoldcf/branches/CONNECTORS-781/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/ServletListener.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/ServletListener.java?rev=1547019&r1=1547018&r2=1547019&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/ServletListener.java
(original)
+++ manifoldcf/branches/CONNECTORS-781/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/ServletListener.java
Mon Dec  2 13:45:49 2013
@@ -20,6 +20,7 @@ package org.apache.manifoldcf.combinedse
 
 import org.apache.manifoldcf.core.interfaces.*;
 import org.apache.manifoldcf.crawler.system.ManifoldCF;
+import org.apache.manifoldcf.agents.system.AgentsDaemon;
 import javax.servlet.*;
 
 /** This class furnishes a servlet shutdown hook for ManifoldCF.  It should be referenced
in the
@@ -59,10 +60,10 @@ public class ServletListener implements 
     {
       if (agentsThread != null)
       {
-        ManifoldCF.assertAgentsShutdownSignal(tc);
+        AgentsDaemon.assertAgentsShutdownSignal(tc);
         agentsThread.finishUp();
         agentsThread = null;
-        ManifoldCF.clearAgentsShutdownSignal(tc);
+        AgentsDaemon.clearAgentsShutdownSignal(tc);
       }
     }
     catch (InterruptedException e)
@@ -94,14 +95,15 @@ public class ServletListener implements 
       IThreadContext tc = ThreadContextFactory.make();
       try
       {
-        ManifoldCF.clearAgentsShutdownSignal(tc);
+        AgentsDaemon.clearAgentsShutdownSignal(tc);
+        AgentsDaemon ad = new AgentsDaemon(processID);
         try
         {
-          ManifoldCF.runAgents(tc, processID);
+          ad.runAgents(tc);
         }
         finally
         {
-          ManifoldCF.stopAgents(tc, processID);
+          ad.stopAgents(tc);
         }
       }
       catch (Throwable e)

Modified: manifoldcf/branches/CONNECTORS-781/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java?rev=1547019&r1=1547018&r2=1547019&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java
(original)
+++ manifoldcf/branches/CONNECTORS-781/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java
Mon Dec  2 13:45:49 2013
@@ -21,6 +21,7 @@ import java.io.*;
 import org.apache.manifoldcf.core.interfaces.*;
 import org.apache.manifoldcf.crawler.system.*;
 import org.apache.manifoldcf.crawler.*;
+import org.apache.manifoldcf.agents.system.AgentsDaemon;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -138,8 +139,9 @@ public class ManifoldCFJettyRunner
   {
     String processID = ManifoldCF.getProcessID();
     // Do this so we don't have to call stopAgents() ourselves.
-    ManifoldCF.registerAgentsShutdownHook(tc, processID);
-    ManifoldCF.runAgents(tc, processID);
+    AgentsDaemon ad = new AgentsDaemon(processID);
+    ad.registerAgentsShutdownHook(tc);
+    ad.runAgents(tc);
   }
 
   /**
@@ -198,7 +200,7 @@ public class ManifoldCFJettyRunner
       if (useParentClassLoader)
       {
         // Clear the agents shutdown signal.
-        ManifoldCF.clearAgentsShutdownSignal(tc);
+        AgentsDaemon.clearAgentsShutdownSignal(tc);
         
         // Do the basic initialization of the database and its schema
         ManifoldCF.createSystemDatabase(tc);

Modified: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java?rev=1547019&r1=1547018&r2=1547019&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java
(original)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java
Mon Dec  2 13:45:49 2013
@@ -22,6 +22,7 @@ import org.apache.manifoldcf.core.interf
 import org.apache.manifoldcf.agents.interfaces.*;
 import org.apache.manifoldcf.crawler.interfaces.*;
 import org.apache.manifoldcf.crawler.system.ManifoldCF;
+import org.apache.manifoldcf.agents.system.AgentsDaemon;
 
 import java.io.*;
 import java.util.*;
@@ -575,7 +576,7 @@ public class ManifoldCFInstance
       // If all worked, then we can start the daemon.
       // Clear the agents shutdown signal.
       IThreadContext tc = ThreadContextFactory.make();
-      ManifoldCF.clearAgentsShutdownSignal(tc);
+      AgentsDaemon.clearAgentsShutdownSignal(tc);
 
       daemonThread = new DaemonThread();
       daemonThread.start();
@@ -670,7 +671,7 @@ public class ManifoldCFInstance
       if (!singleWar)
       {
         // Shut down daemon
-        ManifoldCF.assertAgentsShutdownSignal(tc);
+        AgentsDaemon.assertAgentsShutdownSignal(tc);
         
         // Wait for daemon thread to exit.
         while (true)
@@ -721,9 +722,10 @@ public class ManifoldCFInstance
       IThreadContext tc = ThreadContextFactory.make();
       // Now, start the server, and then wait for the shutdown signal.  On shutdown, we have
to actually do the cleanup,
       // because the JVM isn't going away.
+      AgentsDaemon ad = new AgentsDaemon(processID);
       try
       {
-        ManifoldCF.runAgents(tc, processID);
+        ad.runAgents(tc);
       }
       catch (ManifoldCFException e)
       {
@@ -733,7 +735,7 @@ public class ManifoldCFInstance
       {
         try
         {
-          ManifoldCF.stopAgents(tc, processID);
+          ad.stopAgents(tc);
         }
         catch (ManifoldCFException e)
         {



Mime
View raw message