manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1547167 - in /manifoldcf/trunk: ./ framework/agents/src/main/java/org/apache/manifoldcf/agents/ framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ framework/combined-service/src/main/java/org/apache/manifoldcf/combinedserv...
Date Mon, 02 Dec 2013 19:47:15 GMT
Author: kwright
Date: Mon Dec  2 19:47:15 2013
New Revision: 1547167

URL: http://svn.apache.org/r1547167
Log:
As part of CONNECTORS-781, create at least one multiprocess test, and debug multiprocess code.

Added:
    manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/AgentsDaemon.java
      - copied unchanged from r1547166, manifoldcf/branches/CONNECTORS-781/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/AgentsDaemon.java
Modified:
    manifoldcf/trunk/   (props changed)
    manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentRun.java
    manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentStop.java
    manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java
    manifoldcf/trunk/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/ServletListener.java
    manifoldcf/trunk/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java
    manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/EventManager.java
    manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java
    manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerHSQLDBTest.java
    manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerTester.java

Propchange: manifoldcf/trunk/
------------------------------------------------------------------------------
  Merged /manifoldcf/branches/CONNECTORS-781:r1546969-1547166

Modified: manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentRun.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentRun.java?rev=1547167&r1=1547166&r2=1547167&view=diff
==============================================================================
--- manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentRun.java
(original)
+++ manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentRun.java
Mon Dec  2 19:47:15 2013
@@ -59,8 +59,9 @@ public class AgentRun extends BaseAgents
       
       Logging.root.info("Running...");
       // Register hook first so stopAgents() not required
-      ManifoldCF.registerAgentsShutdownHook(tc, processID);
-      ManifoldCF.runAgents(tc, processID);
+      AgentsDaemon ad = new AgentsDaemon(processID);
+      ad.registerAgentsShutdownHook(tc);
+      ad.runAgents(tc);
       Logging.root.info("Shutting down...");
     }
     catch (ManifoldCFException e)

Modified: manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentStop.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentStop.java?rev=1547167&r1=1547166&r2=1547167&view=diff
==============================================================================
--- manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentStop.java
(original)
+++ manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/AgentStop.java
Mon Dec  2 19:47:15 2013
@@ -37,7 +37,7 @@ public class AgentStop extends BaseAgent
     // As part of the work for CONNECTORS-781, this method is now synchronous.
     // We assert the shutdown signal, and then wait until all active services have shut down.
     ILockManager lockManager = LockManagerFactory.make(tc);
-    ManifoldCF.assertAgentsShutdownSignal(tc);
+    AgentsDaemon.assertAgentsShutdownSignal(tc);
     try
     {
       Logging.root.info("Shutdown signal sent");
@@ -61,7 +61,7 @@ public class AgentStop extends BaseAgent
     finally
     {
       // Clear shutdown signal
-      ManifoldCF.clearAgentsShutdownSignal(tc);
+      AgentsDaemon.clearAgentsShutdownSignal(tc);
     }
   }
 

Modified: manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java?rev=1547167&r1=1547166&r2=1547167&view=diff
==============================================================================
--- manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java
(original)
+++ manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ManifoldCF.java
Mon Dec  2 19:47:15 2013
@@ -32,11 +32,6 @@ public class ManifoldCF extends org.apac
   // Agents initialized flag
   protected static boolean agentsInitialized = false;
   
-  /** This is the place we keep track of the agents we've started. */
-  protected static Map<String,IAgent> runningHash = new HashMap<String,IAgent>();
-  /** This flag prevents startAgents() from starting anything once stopAgents() has been
called. */
-  protected static boolean stopAgentsRun = false;
-  
   /** Initialize environment.
   */
   public static void initializeEnvironment(IThreadContext threadContext)
@@ -88,10 +83,6 @@ public class ManifoldCF extends org.apac
     synchronized (initializeFlagLock)
     {
       org.apache.manifoldcf.core.system.ManifoldCF.resetEnvironment(threadContext);
-      synchronized (runningHash)
-      {
-        stopAgentsRun = false;
-      }
     }
   }
 
@@ -128,317 +119,6 @@ public class ManifoldCF extends org.apac
     mgr.deinstall();
   }
 
-  // There are a number of different ways of running the agents framework.
-  // (1) Repeatedly call checkAgents(), and when all done make sure to call stopAgents().
-  // (2) Call registerAgentsShutdownHook(), then repeatedly run checkAgents(),  Agent shutdown
happens on JVM exit.
-  // (3) Call runAgents(), which will wait for someone else to call assertAgentsShutdownSignal().
 Before exit, stopAgents() must be called.
-  // (4) Call registerAgentsShutdownHook(), then call runAgents(), which will wait for someone
else to call assertAgentsShutdownSignal().  Shutdown happens on JVM exit.
-  
-  /** Assert shutdown signal.
-  */
-  public static void assertAgentsShutdownSignal(IThreadContext threadContext)
-    throws ManifoldCFException
-  {
-    ILockManager lockManager = LockManagerFactory.make(threadContext);
-    lockManager.setGlobalFlag(agentShutdownSignal);
-  }
-  
-  /** Clear shutdown signal.
-  */
-  public static void clearAgentsShutdownSignal(IThreadContext threadContext)
-    throws ManifoldCFException
-  {
-    ILockManager lockManager = LockManagerFactory.make(threadContext);
-    lockManager.clearGlobalFlag(agentShutdownSignal);
-  }
-
-
-  /** Register agents shutdown hook.
-  * Call this ONCE before calling startAgents or checkAgents the first time, if you want
automatic cleanup of agents on JVM stop.
-  */
-  public static void registerAgentsShutdownHook(IThreadContext threadContext, String processID)
-    throws ManifoldCFException
-  {
-    // Create the shutdown hook for agents.  All activity will be keyed off of runningHash,
so it is safe to do this under all conditions.
-    org.apache.manifoldcf.core.system.ManifoldCF.addShutdownHook(new AgentsShutdownHook(processID));
-  }
-  
-  /** Agent service name prefix (followed by agent class name) */
-  public static final String agentServicePrefix = "AGENT_";
-  
-  protected static AgentsThread agentsThread = null;
-
-  /** Run agents process.
-  * This method will not return until a shutdown signal is sent.
-  */
-  public static void runAgents(IThreadContext threadContext, String processID)
-    throws ManifoldCFException
-  {
-    ILockManager lockManager = LockManagerFactory.make(threadContext);
-
-    // Don't come up at all if shutdown signal in force
-    if (lockManager.checkGlobalFlag(agentShutdownSignal))
-      return;
-
-    // Create and start agents thread.
-    startAgents(threadContext, processID);
-    
-    while (true)
-    {
-      // Any shutdown signal yet?
-      if (lockManager.checkGlobalFlag(agentShutdownSignal))
-        break;
-          
-      try
-      {
-        ManifoldCF.sleep(5000L);
-      }
-      catch (InterruptedException e)
-      {
-        break;
-      }
-    }
-    
-  }
-
-  /** Start agents thread.
-  */
-  public static void startAgents(IThreadContext threadContext, String processID)
-    throws ManifoldCFException
-  {
-    // Create and start agents thread.
-    agentsThread = new AgentsThread(processID);
-    agentsThread.start();
-  }
-  
-  /** Stop all started agents.
-  */
-  public static void stopAgents(IThreadContext threadContext, String processID)
-    throws ManifoldCFException
-  {
-    // Shut down agents background thread.
-    while (agentsThread != null)
-    {
-      agentsThread.interrupt();
-      if (!agentsThread.isAlive())
-        agentsThread = null;
-    }
-    
-    // Shut down running agents services directly.
-    ILockManager lockManager = LockManagerFactory.make(threadContext);
-    synchronized (runningHash)
-    {
-      // This is supposedly safe; iterator remove is used
-      Iterator<String> iter = runningHash.keySet().iterator();
-      while (iter.hasNext())
-      {
-        String className = iter.next();
-        IAgent agent = runningHash.get(className);
-        // Stop it
-        agent.stopAgent(threadContext);
-        lockManager.endServiceActivity(getAgentsClassServiceType(className), processID);
-        iter.remove();
-        agent.cleanUp(threadContext);
-      }
-    }
-    // Done.
-  }
-
-  protected static String getAgentsClassServiceType(String agentClassName)
-  {
-    return agentServicePrefix + agentClassName;
-  }
-  
-  /** Agents thread.  This runs in background until interrupted, at which point
-  * it shuts down.  Its responsibilities include cleaning up after dead processes,
-  * as well as starting newly-registered agent processes, and terminating ones that disappear.
-  */
-  protected static class AgentsThread extends Thread
-  {
-    protected final String processID;
-    
-    public AgentsThread(String processID)
-    {
-      super();
-      this.processID = processID;
-      setName("Agents thread");
-      setDaemon(true);
-    }
-    
-    public void run()
-    {
-      try
-      {
-        IThreadContext threadContext = ThreadContextFactory.make();
-        while (true)
-        {
-          try
-          {
-            if (Thread.currentThread().isInterrupted())
-              throw new ManifoldCFException("Interrupted",ManifoldCFException.INTERRUPTED);
-
-            checkAgents(threadContext, processID);
-            ManifoldCF.sleep(5000L);
-          }
-          catch (InterruptedException e)
-          {
-            break;
-          }
-          catch (ManifoldCFException e)
-          {
-            if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
-              break;
-            Logging.agents.error("Exception tossed: "+e.getMessage(),e);
-          }
-          catch (OutOfMemoryError e)
-          {
-            System.err.println("Agents process ran out of memory - shutting down");
-            e.printStackTrace(System.err);
-            System.exit(-200);
-          }
-          catch (Throwable e)
-          {
-            Logging.agents.fatal("Error tossed: "+e.getMessage(),e);
-          }
-        }
-      }
-      catch (Throwable e)
-      {
-        // Severe error on initialization
-        System.err.println("Agents process could not start - shutting down");
-        Logging.agents.fatal("AgentThread initialization error tossed: "+e.getMessage(),e);
-        System.exit(-300);
-      }
-    }
-  }
-
-  /** Start all not-running agents.
-  *@param threadContext is the thread context.
-  */
-  protected static void checkAgents(IThreadContext threadContext, String processID)
-    throws ManifoldCFException
-  {
-    ILockManager lockManager = LockManagerFactory.make(threadContext);
-    // Get agent manager
-    IAgentManager manager = AgentManagerFactory.make(threadContext);
-    synchronized (runningHash)
-    {
-      String[] classes = manager.getAllAgents();
-      Set<String> currentAgentClasses = new HashSet<String>();
-
-      int i = 0;
-      while (i < classes.length)
-      {
-        String className = classes[i++];
-        if (runningHash.get(className) == null)
-        {
-          // Start this agent
-          IAgent agent = AgentFactory.make(className);
-          agent.initialize(threadContext);
-          try
-          {
-            // Throw a lock, so that cleanup processes and startup processes don't collide.
-            String serviceType = getAgentsClassServiceType(className);
-            lockManager.registerServiceBeginServiceActivity(serviceType, processID, new CleanupAgent(threadContext,
agent, processID));
-            // There is a potential race condition where the agent has been started but hasn't
yet appeared in runningHash.
-            // But having runningHash be the synchronizer for this activity will prevent
any problems.
-            agent.startAgent(threadContext, processID);
-            // Successful!
-            runningHash.put(className,agent);
-          }
-          catch (ManifoldCFException e)
-          {
-            if (e.getErrorCode() != ManifoldCFException.INTERRUPTED)
-              agent.cleanUp(threadContext);
-            throw e;
-          }
-        }
-        currentAgentClasses.add(className);
-      }
-
-      // Go through running hash and look for agents processes that have left
-      Iterator<String> runningAgentsIterator = runningHash.keySet().iterator();
-      while (runningAgentsIterator.hasNext())
-      {
-        String runningAgentClass = runningAgentsIterator.next();
-        if (!currentAgentClasses.contains(runningAgentClass))
-        {
-          // Shut down this one agent.
-          IAgent agent = runningHash.get(runningAgentClass);
-          // Stop it
-          agent.stopAgent(threadContext);
-          lockManager.endServiceActivity(getAgentsClassServiceType(runningAgentClass), processID);
-          runningAgentsIterator.remove();
-          agent.cleanUp(threadContext);
-        }
-      }
-    }
-
-    synchronized (runningHash)
-    {
-      // For every class we're supposed to be running, find registered but no-longer-active
instances and clean
-      // up after them.
-      for (String agentsClass : runningHash.keySet())
-      {
-        IAgent agent = runningHash.get(agentsClass);
-        IServiceCleanup cleanup = new CleanupAgent(threadContext, agent, processID);
-        String agentsClassServiceType = getAgentsClassServiceType(agentsClass);
-        while (!lockManager.cleanupInactiveService(agentsClassServiceType, cleanup))
-        {
-          // Loop until no more inactive services
-        }
-      }
-    }
-    
-  }
-
-  
-  protected static class CleanupAgent implements IServiceCleanup
-  {
-    protected final IAgent agent;
-    protected final IThreadContext threadContext;
-    protected final String processID;
-
-    public CleanupAgent(IThreadContext threadContext, IAgent agent, String processID)
-    {
-      this.agent = agent;
-      this.threadContext = threadContext;
-      this.processID = processID;
-    }
-    
-    /** Clean up after the specified service.  This method will block any startup of the
specified
-    * service for as long as it runs.
-    *@param serviceName is the name of the service.
-    */
-    @Override
-    public void cleanUpService(String serviceName)
-      throws ManifoldCFException
-    {
-      agent.cleanUpAgentData(threadContext, processID, serviceName);
-    }
-
-    /** Clean up after ALL services of the type on the cluster.
-    */
-    @Override
-    public void cleanUpAllServices()
-      throws ManifoldCFException
-    {
-      agent.cleanUpAllAgentData(threadContext, processID);
-    }
-    
-    /** Perform cluster initialization - that is, whatever is needed presuming that the
-    * cluster has been down for an indeterminate period of time, but is otherwise in a clean
-    * state.
-    */
-    @Override
-    public void clusterInit()
-      throws ManifoldCFException
-    {
-      agent.clusterInit(threadContext);
-    }
-
-  }
-  
   /** Signal output connection needs redoing.
   * This is called when something external changed on an output connection, and
   * therefore all associated documents must be reindexed.
@@ -456,30 +136,6 @@ public class ManifoldCF extends org.apac
     AgentManagerFactory.noteOutputConnectionChange(threadContext,connectionName);
   }
   
-  /** Agents shutdown hook class */
-  protected static class AgentsShutdownHook implements IShutdownHook
-  {
-    protected final String processID;
-
-    public AgentsShutdownHook(String processID)
-    {
-      this.processID = processID;
-    }
-    
-    public void doCleanup()
-      throws ManifoldCFException
-    {
-      // Shutting down in this way must prevent startup from taking place.
-      synchronized (runningHash)
-      {
-        stopAgentsRun = true;
-      }
-      IThreadContext tc = ThreadContextFactory.make();
-      stopAgents(tc,processID);
-    }
-    
-  }
-  
   // Helper methods for API support.  These are made public so connectors can use them to
implement the executeCommand method.
   
   // These are the universal node types.

Modified: manifoldcf/trunk/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/ServletListener.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/ServletListener.java?rev=1547167&r1=1547166&r2=1547167&view=diff
==============================================================================
--- manifoldcf/trunk/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/ServletListener.java
(original)
+++ manifoldcf/trunk/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/ServletListener.java
Mon Dec  2 19:47:15 2013
@@ -20,6 +20,7 @@ package org.apache.manifoldcf.combinedse
 
 import org.apache.manifoldcf.core.interfaces.*;
 import org.apache.manifoldcf.crawler.system.ManifoldCF;
+import org.apache.manifoldcf.agents.system.AgentsDaemon;
 import javax.servlet.*;
 
 /** This class furnishes a servlet shutdown hook for ManifoldCF.  It should be referenced
in the
@@ -59,10 +60,10 @@ public class ServletListener implements 
     {
       if (agentsThread != null)
       {
-        ManifoldCF.assertAgentsShutdownSignal(tc);
+        AgentsDaemon.assertAgentsShutdownSignal(tc);
         agentsThread.finishUp();
         agentsThread = null;
-        ManifoldCF.clearAgentsShutdownSignal(tc);
+        AgentsDaemon.clearAgentsShutdownSignal(tc);
       }
     }
     catch (InterruptedException e)
@@ -94,14 +95,15 @@ public class ServletListener implements 
       IThreadContext tc = ThreadContextFactory.make();
       try
       {
-        ManifoldCF.clearAgentsShutdownSignal(tc);
+        AgentsDaemon.clearAgentsShutdownSignal(tc);
+        AgentsDaemon ad = new AgentsDaemon(processID);
         try
         {
-          ManifoldCF.runAgents(tc, processID);
+          ad.runAgents(tc);
         }
         finally
         {
-          ManifoldCF.stopAgents(tc, processID);
+          ad.stopAgents(tc);
         }
       }
       catch (Throwable e)

Modified: manifoldcf/trunk/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java?rev=1547167&r1=1547166&r2=1547167&view=diff
==============================================================================
--- manifoldcf/trunk/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java
(original)
+++ manifoldcf/trunk/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyRunner.java
Mon Dec  2 19:47:15 2013
@@ -21,6 +21,7 @@ import java.io.*;
 import org.apache.manifoldcf.core.interfaces.*;
 import org.apache.manifoldcf.crawler.system.*;
 import org.apache.manifoldcf.crawler.*;
+import org.apache.manifoldcf.agents.system.AgentsDaemon;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -138,8 +139,9 @@ public class ManifoldCFJettyRunner
   {
     String processID = ManifoldCF.getProcessID();
     // Do this so we don't have to call stopAgents() ourselves.
-    ManifoldCF.registerAgentsShutdownHook(tc, processID);
-    ManifoldCF.runAgents(tc, processID);
+    AgentsDaemon ad = new AgentsDaemon(processID);
+    ad.registerAgentsShutdownHook(tc);
+    ad.runAgents(tc);
   }
 
   /**
@@ -198,7 +200,7 @@ public class ManifoldCFJettyRunner
       if (useParentClassLoader)
       {
         // Clear the agents shutdown signal.
-        ManifoldCF.clearAgentsShutdownSignal(tc);
+        AgentsDaemon.clearAgentsShutdownSignal(tc);
         
         // Do the basic initialization of the database and its schema
         ManifoldCF.createSystemDatabase(tc);

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/EventManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/EventManager.java?rev=1547167&r1=1547166&r2=1547167&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/EventManager.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/EventManager.java
Mon Dec  2 19:47:15 2013
@@ -127,7 +127,7 @@ public class EventManager extends org.ap
     ArrayList list = new ArrayList();
     String query = buildConjunctionClause(list,new ClauseDescription[]{
       new UnitaryClause(processIDField,processID)});
-    performDelete("WHERE "+query,null,null);
+    performDelete("WHERE "+query,list,null);
   }
 
   /** Clean up after all processIDs.

Modified: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java?rev=1547167&r1=1547166&r2=1547167&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/ManifoldCFInstance.java
Mon Dec  2 19:47:15 2013
@@ -22,6 +22,7 @@ import org.apache.manifoldcf.core.interf
 import org.apache.manifoldcf.agents.interfaces.*;
 import org.apache.manifoldcf.crawler.interfaces.*;
 import org.apache.manifoldcf.crawler.system.ManifoldCF;
+import org.apache.manifoldcf.agents.system.AgentsDaemon;
 
 import java.io.*;
 import java.util.*;
@@ -56,39 +57,57 @@ import org.apache.http.entity.ContentTyp
 /** Tests that run the "agents daemon" should be derived from this */
 public class ManifoldCFInstance
 {
-  protected boolean webapps = true;
-  protected boolean singleWar = false;
-  protected int testPort = 8346;
+  protected final boolean webapps;
+  protected final boolean singleWar;
+  protected final int testPort;
+  protected final String processID;
   
   protected DaemonThread daemonThread = null;
   protected Server server = null;
 
   public ManifoldCFInstance()
   {
+    this(ManifoldCF.getProcessID(), 8346, false, true);
   }
   
+  public ManifoldCFInstance(String processID)
+  {
+    this(processID, 8346, false, true);
+  }
+
   public ManifoldCFInstance(boolean singleWar)
   {
-    this(8346,singleWar,true);
+    this(ManifoldCF.getProcessID(), 8346, singleWar, true);
+  }
+  
+  public ManifoldCFInstance(String processID, boolean singleWar)
+  {
+    this(processID, 8346, singleWar, true);
   }
 
   public ManifoldCFInstance(boolean singleWar, boolean webapps)
   {
-    this(8346,singleWar,webapps);
+    this(ManifoldCF.getProcessID(), 8346, singleWar, webapps);
+  }
+
+  public ManifoldCFInstance(String processID, boolean singleWar, boolean webapps)
+  {
+    this(processID, 8346, singleWar, webapps);
   }
   
-  public ManifoldCFInstance(int testPort)
+  public ManifoldCFInstance(String processID, int testPort)
   {
-    this(testPort,false,true);
+    this(processID, testPort, false, true);
   }
 
-  public ManifoldCFInstance(int testPort, boolean singleWar)
+  public ManifoldCFInstance(String processID, int testPort, boolean singleWar)
   {
-    this(testPort,singleWar,true);
+    this(processID, testPort, singleWar, true);
   }
 
-  public ManifoldCFInstance(int testPort, boolean singleWar, boolean webapps)
+  public ManifoldCFInstance(String processID, int testPort, boolean singleWar, boolean webapps)
   {
+    this.processID = processID;
     this.webapps = webapps;
     this.testPort = testPort;
     this.singleWar = singleWar;
@@ -575,9 +594,9 @@ public class ManifoldCFInstance
       // If all worked, then we can start the daemon.
       // Clear the agents shutdown signal.
       IThreadContext tc = ThreadContextFactory.make();
-      ManifoldCF.clearAgentsShutdownSignal(tc);
+      AgentsDaemon.clearAgentsShutdownSignal(tc);
 
-      daemonThread = new DaemonThread();
+      daemonThread = new DaemonThread(processID);
       daemonThread.start();
     }
   }
@@ -667,30 +686,48 @@ public class ManifoldCFInstance
         }
       }
 
-      if (!singleWar)
+      try
+      {
+        stopNoCleanup();
+      }
+      catch (Exception e)
       {
-        // Shut down daemon
-        ManifoldCF.assertAgentsShutdownSignal(tc);
+        if (currentException == null)
+          currentException = e;
+      }
+    }
+  }
+  
+  public void stopNoCleanup()
+    throws Exception
+  {
+    if (daemonThread != null)
+    {
+      Exception currentException = null;
+      
+      // Shut down daemon - but only ONE daemon
+      //AgentsDaemon.assertAgentsShutdownSignal(tc);
         
-        // Wait for daemon thread to exit.
-        while (true)
+      // Wait for daemon thread to exit.
+      while (true)
+      {
+        daemonThread.interrupt();
+        if (daemonThread.isAlive())
         {
-          if (daemonThread.isAlive())
-          {
-            Thread.sleep(1000L);
-            continue;
-          }
-          break;
+          Thread.sleep(1000L);
+          continue;
         }
-
-        Exception e = daemonThread.getDaemonException();
-        if (e != null)
-          currentException = e;
+        break;
       }
-        
+
+      Exception e = daemonThread.getDaemonException();
+      if (e != null || !(e instanceof InterruptedException))
+        currentException = e;
+
       if (currentException != null)
         throw currentException;
     }
+        
   }
   
   public void unload()
@@ -708,22 +745,24 @@ public class ManifoldCFInstance
   
   protected static class DaemonThread extends Thread
   {
+    protected final String processID;
     protected Exception daemonException = null;
     
-    public DaemonThread()
+    public DaemonThread(String processID)
     {
+      this.processID = processID;
       setName("Daemon thread");
     }
     
     public void run()
     {
-      String processID = ManifoldCF.getProcessID();
       IThreadContext tc = ThreadContextFactory.make();
       // Now, start the server, and then wait for the shutdown signal.  On shutdown, we have
to actually do the cleanup,
       // because the JVM isn't going away.
+      AgentsDaemon ad = new AgentsDaemon(processID);
       try
       {
-        ManifoldCF.runAgents(tc, processID);
+        ad.runAgents(tc);
       }
       catch (ManifoldCFException e)
       {
@@ -733,7 +772,7 @@ public class ManifoldCFInstance
       {
         try
         {
-          ManifoldCF.stopAgents(tc, processID);
+          ad.stopAgents(tc);
         }
         catch (ManifoldCFException e)
         {

Modified: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerHSQLDBTest.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerHSQLDBTest.java?rev=1547167&r1=1547166&r2=1547167&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerHSQLDBTest.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerHSQLDBTest.java
Mon Dec  2 19:47:15 2013
@@ -28,15 +28,18 @@ import org.junit.*;
 
 /** This is a test of the scheduler.  If the test succeeds, it is because
 * the scheduler has properly distributed requests for all bins evenly. */
-public class SchedulerHSQLDBTest extends BaseITHSQLDB
+public class SchedulerHSQLDBTest extends ConnectorBaseHSQLDB
 {
-  
+  protected final ManifoldCFInstance mcfInstance1;
+  protected final ManifoldCFInstance mcfInstance2;
   protected SchedulerTester tester;
 
   public SchedulerHSQLDBTest()
   {
-    super(false,false);
-    tester = new SchedulerTester(mcfInstance);
+    super();
+    mcfInstance1 = new ManifoldCFInstance("A",false,false);
+    mcfInstance2 = new ManifoldCFInstance("B",false,false);
+    tester = new SchedulerTester(mcfInstance1,mcfInstance2);
   }
   
   @Override
@@ -70,5 +73,68 @@ public class SchedulerHSQLDBTest extends
     tester.executeTest();
   }
   
+  @Before
+  public void setUp()
+    throws Exception
+  {
+    initializeSystem();
+    try
+    {
+      localReset();
+    }
+    catch (Exception e)
+    {
+      System.out.println("Warning: Preclean failed: "+e.getMessage());
+    }
+    try
+    {
+      localSetUp();
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+  
+  @After
+  public void cleanUp()
+    throws Exception
+  {
+    Exception currentException = null;
+    // Last, shut down the web applications.
+    // If this is done too soon it closes the database before the rest of the cleanup happens.
+    try
+    {
+      mcfInstance1.unload();
+    }
+    catch (Exception e)
+    {
+      if (currentException == null)
+        currentException = e;
+    }
+    try
+    {
+      mcfInstance2.unload();
+    }
+    catch (Exception e)
+    {
+      if (currentException == null)
+        currentException = e;
+    }
+    try
+    {
+      localCleanUp();
+    }
+    catch (Exception e)
+    {
+      e.printStackTrace();
+      throw e;
+    }
+    if (currentException != null)
+      throw currentException;
+    cleanupSystem();
+  }
+  
 
 }

Modified: manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerTester.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerTester.java?rev=1547167&r1=1547166&r2=1547167&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerTester.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/test/java/org/apache/manifoldcf/crawler/tests/SchedulerTester.java
Mon Dec  2 19:47:15 2013
@@ -29,16 +29,20 @@ import java.util.*;
 /** This is a very basic sanity check */
 public class SchedulerTester
 {
-  protected ManifoldCFInstance instance;
+  protected final ManifoldCFInstance instance1;
+  protected final ManifoldCFInstance instance2;
   
-  public SchedulerTester(ManifoldCFInstance instance)
+  public SchedulerTester(ManifoldCFInstance instance1, ManifoldCFInstance instance2)
   {
-    this.instance = instance;
+    this.instance1 = instance1;
+    this.instance2 = instance2;
   }
   
   public void executeTest()
     throws Exception
   {
+    instance1.start();
+    
     // Hey, we were able to install the file system connector etc.
     // Now, create a local test job and run it.
     IThreadContext tc = ThreadContextFactory.make();
@@ -78,20 +82,31 @@ public class SchedulerTester
 
     // Now, start the job, and wait until it is running.
     jobManager.manualStart(job.getID());
-    instance.waitJobRunningNative(jobManager,job.getID(),30000L);
+    instance1.waitJobRunningNative(jobManager,job.getID(),30000L);
+    
+    // Start the second instance.
+    instance2.start();
+    // Wait long enough for the stuffing etc to take place once
+    Thread.sleep(5000L);
+    // Terminate instance1.  Instance2 should keep going.
+    instance1.stopNoCleanup();
     
     // Wait for the job to become inactive.  The time should be at least long enough to handle
     // 100 documents per bin, but not significantly greater than that.  Let's say 120 seconds.
     long startTime = System.currentTimeMillis();
-    instance.waitJobInactiveNative(jobManager,job.getID(),150000L);
+    instance2.waitJobInactiveNative(jobManager,job.getID(),150000L);
     long endTime = System.currentTimeMillis();
     if (jobManager.getStatus(job.getID()).getDocumentsProcessed() != 10+10*200)
       throw new Exception("Expected 2010 documents, saw "+jobManager.getStatus(job.getID()).getDocumentsProcessed());
     if (endTime-startTime < 96000L)
       throw new Exception("Job finished too quickly; throttling clearly failed");
     System.out.println("Crawl took "+(endTime-startTime)+" milliseconds");
+    
     // Now, delete the job.
     jobManager.deleteJob(job.getID());
-    instance.waitJobDeletedNative(jobManager,job.getID(),120000L);
+    instance2.waitJobDeletedNative(jobManager,job.getID(),120000L);
+
+    // Shut down instance2
+    instance2.stop();
   }
 }



Mime
View raw message