manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1640019 - in /manifoldcf/branches/dev_1x: ./ framework/ framework/agents/src/main/java/org/apache/manifoldcf/agents/system/ framework/api-service/src/main/java/org/apache/manifoldcf/apiservice/ framework/combined-service/src/main/java/org/...
Date Sun, 16 Nov 2014 17:48:07 GMT
Author: kwright
Date: Sun Nov 16 17:48:07 2014
New Revision: 1640019

URL: http://svn.apache.org/r1640019
Log:
Pull up polling event registry code from trunk

Added:
    manifoldcf/branches/dev_1x/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IPollingHook.java
      - copied unchanged from r1640018, manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IPollingHook.java
Modified:
    manifoldcf/branches/dev_1x/   (props changed)
    manifoldcf/branches/dev_1x/framework/   (props changed)
    manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java
    manifoldcf/branches/dev_1x/framework/api-service/src/main/java/org/apache/manifoldcf/apiservice/IdleCleanupThread.java
    manifoldcf/branches/dev_1x/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/IdleCleanupThread.java
    manifoldcf/branches/dev_1x/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
    manifoldcf/branches/dev_1x/framework/crawler-ui/src/main/java/org/apache/manifoldcf/crawlerui/IdleCleanupThread.java
    manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/system/IdleCleanupThread.java
    manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java

Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
  Merged /manifoldcf/trunk:r1640018

Propchange: manifoldcf/branches/dev_1x/framework/
------------------------------------------------------------------------------
  Merged /manifoldcf/trunk/framework:r1640018

Modified: manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java?rev=1640019&r1=1640018&r2=1640019&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java
(original)
+++ manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/system/IdleCleanupThread.java
Sun Nov 16 17:48:07 2014
@@ -51,14 +51,10 @@ public class IdleCleanupThread extends T
     {
       // Create a thread context object.
       IThreadContext threadContext = ThreadContextFactory.make();
-      // Get the cache handle.
-      ICacheManager cacheManager = CacheManagerFactory.make(threadContext);
       // Get the output connector pool handle
       IOutputConnectorPool outputConnectorPool = OutputConnectorPoolFactory.make(threadContext);
       // Get the transformation connector pool handle
       ITransformationConnectorPool transformationConnectorPool = TransformationConnectorPoolFactory.make(threadContext);
-      // Throttler subsystem
-      IThrottleGroups throttleGroups = ThrottleGroupsFactory.make(threadContext);
       
       /* For HSQLDB debugging...
       IDBInterface database = DBInterfaceFactory.make(threadContext,
@@ -93,10 +89,8 @@ public class IdleCleanupThread extends T
           // Do the cleanup
           outputConnectorPool.pollAllConnectors();
           transformationConnectorPool.pollAllConnectors();
-          // Poll connection bins
-          throttleGroups.poll();
-          // Expire objects
-          cacheManager.expireObjects(System.currentTimeMillis());
+          // Poll all basic services
+          ManifoldCF.pollAll(threadContext);
           
           // Sleep for the retry interval.
           ManifoldCF.sleep(5000L);

Modified: manifoldcf/branches/dev_1x/framework/api-service/src/main/java/org/apache/manifoldcf/apiservice/IdleCleanupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/api-service/src/main/java/org/apache/manifoldcf/apiservice/IdleCleanupThread.java?rev=1640019&r1=1640018&r2=1640019&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/api-service/src/main/java/org/apache/manifoldcf/apiservice/IdleCleanupThread.java
(original)
+++ manifoldcf/branches/dev_1x/framework/api-service/src/main/java/org/apache/manifoldcf/apiservice/IdleCleanupThread.java
Sun Nov 16 17:48:07 2014
@@ -51,16 +51,12 @@ public class IdleCleanupThread extends T
     {
       // Create a thread context object.
       IThreadContext threadContext = ThreadContextFactory.make();
-      // Get the cache handle.
-      ICacheManager cacheManager = CacheManagerFactory.make(threadContext);
       
       IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
       IOutputConnectorPool outputConnectorPool = OutputConnectorPoolFactory.make(threadContext);
       ITransformationConnectorPool transformationConnectorPool = TransformationConnectorPoolFactory.make(threadContext);
       IAuthorityConnectorPool authorityConnectorPool = AuthorityConnectorPoolFactory.make(threadContext);
       IMappingConnectorPool mappingConnectorPool = MappingConnectorPoolFactory.make(threadContext);
-      
-      IThrottleGroups throttleGroups = ThrottleGroupsFactory.make(threadContext);
 
       // Loop
       while (true)
@@ -74,10 +70,8 @@ public class IdleCleanupThread extends T
           transformationConnectorPool.pollAllConnectors();
           authorityConnectorPool.pollAllConnectors();
           mappingConnectorPool.pollAllConnectors();
-          
-          throttleGroups.poll();
-          
-          cacheManager.expireObjects(System.currentTimeMillis());
+          // Poll all basic services
+          ManifoldCF.pollAll(threadContext);
           
           // Sleep for the retry interval.
           ManifoldCF.sleep(5000L);

Modified: manifoldcf/branches/dev_1x/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/IdleCleanupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/IdleCleanupThread.java?rev=1640019&r1=1640018&r2=1640019&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/IdleCleanupThread.java
(original)
+++ manifoldcf/branches/dev_1x/framework/combined-service/src/main/java/org/apache/manifoldcf/combinedservice/IdleCleanupThread.java
Sun Nov 16 17:48:07 2014
@@ -51,16 +51,12 @@ public class IdleCleanupThread extends T
     {
       // Create a thread context object.
       IThreadContext threadContext = ThreadContextFactory.make();
-      // Get the cache handle.
-      ICacheManager cacheManager = CacheManagerFactory.make(threadContext);
       
       IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
       IOutputConnectorPool outputConnectorPool = OutputConnectorPoolFactory.make(threadContext);
       ITransformationConnectorPool transformationConnectorPool = TransformationConnectorPoolFactory.make(threadContext);
       IAuthorityConnectorPool authorityConnectorPool = AuthorityConnectorPoolFactory.make(threadContext);
       IMappingConnectorPool mappingConnectorPool = MappingConnectorPoolFactory.make(threadContext);
-      
-      IThrottleGroups throttleGroups = ThrottleGroupsFactory.make(threadContext);
 
       // Loop
       while (true)
@@ -74,11 +70,9 @@ public class IdleCleanupThread extends T
           transformationConnectorPool.pollAllConnectors();
           authorityConnectorPool.pollAllConnectors();
           mappingConnectorPool.pollAllConnectors();
-          
-          throttleGroups.poll();
-          
-          cacheManager.expireObjects(System.currentTimeMillis());
-          
+          // Poll all basic services
+          ManifoldCF.pollAll(threadContext);
+
           // Sleep for the retry interval.
           ManifoldCF.sleep(5000L);
         }

Modified: manifoldcf/branches/dev_1x/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java?rev=1640019&r1=1640018&r2=1640019&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
(original)
+++ manifoldcf/branches/dev_1x/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
Sun Nov 16 17:48:07 2014
@@ -54,7 +54,10 @@ public class ManifoldCF
   protected static DatabaseShutdown dbShutdown = null;
   
   /** Array of cleanup hooks (for managing shutdown) */
-  protected static ArrayList cleanupHooks = new ArrayList(); 
+  protected final static List<IShutdownHook> cleanupHooks = new ArrayList<IShutdownHook>();

+  
+  /** Array of polling hooks (for managing polling) */
+  protected final static List<IPollingHook> pollingHooks = new ArrayList<IPollingHook>();
   
   /** Shutdown thread */
   protected static Thread shutdownThread;
@@ -254,8 +257,13 @@ public class ManifoldCF
           masterDatabaseUsername = LockManagerFactory.getStringProperty(threadContext,masterDatabaseUsernameProperty,"manifoldcf");
           masterDatabasePassword = LockManagerFactory.getPossiblyObfuscatedStringProperty(threadContext,masterDatabasePasswordProperty,"local_pg_passwd");
 
-          // Register the throttler for cleanup on shutdown
+          // Register the throttler
           addShutdownHook(new ThrottlerShutdown());
+          addPollingHook(new ThrottlerPoll());
+
+          // Put the cache manager in the polling loop
+          addPollingHook(new CachePoll());
+
           // Register the file tracker for cleanup on shutdown
           tracker = new FileTrack();
           addShutdownHook(tracker);
@@ -1247,6 +1255,29 @@ public class ManifoldCF
       cleanupHooks.add(hook);
     }
   }
+
+  /** Add a polling hook to the list.  These hooks will be evaluated in the
+  * order they were added.
+  *@param hook is the polling hook that needs to be added to the sequence.
+  */
+  public static void addPollingHook(IPollingHook hook)
+  {
+    synchronized (pollingHooks)
+    {
+      pollingHooks.add(hook);
+    }
+  }
+  
+  /** Poll all the registered polling services.
+  */
+  public static void pollAll(IThreadContext threadContext)
+    throws ManifoldCFException
+  {
+    for (IPollingHook hook : pollingHooks)
+    {
+      hook.doPoll(threadContext);
+    }
+  }
   
   /** Create a new resource loader based on the default one.  This is used by
   * connectors wishing to make their own resource loaders for isolation purposes.
@@ -1289,7 +1320,7 @@ public class ManifoldCF
           while (i > 0)
           {
             i--;
-            IShutdownHook hook = (IShutdownHook)cleanupHooks.get(i);
+            IShutdownHook hook = cleanupHooks.get(i);
             try
             {
               hook.doCleanup(threadContext);
@@ -1372,6 +1403,22 @@ public class ManifoldCF
 
   }
 
+  /** Class that polls throttler */
+  protected static class ThrottlerPoll implements IPollingHook
+  {
+    public ThrottlerPoll()
+    {
+    }
+    
+    @Override
+    public void doPoll(IThreadContext threadContext)
+      throws ManifoldCFException
+    {
+      IThrottleGroups connectionThrottler = ThrottleGroupsFactory.make(threadContext);
+      connectionThrottler.poll();
+    }
+  }
+  
   /** Class that cleans up throttler on exit */
   protected static class ThrottlerShutdown implements IShutdownHook
   {
@@ -1404,6 +1451,23 @@ public class ManifoldCF
 
   }
   
+  /** Class that cleans up expired cache objects on polling.
+  */
+  protected static class CachePoll implements IPollingHook
+  {
+    public CachePoll()
+    {
+    }
+    
+    @Override
+    public void doPoll(IThreadContext threadContext)
+      throws ManifoldCFException
+    {
+      ICacheManager cacheManager = CacheManagerFactory.make(threadContext);
+      cacheManager.expireObjects(System.currentTimeMillis());
+    }
+  }
+  
   /** Class that cleans up database handles on exit */
   protected static class DatabaseShutdown implements IShutdownHook
   {

Modified: manifoldcf/branches/dev_1x/framework/crawler-ui/src/main/java/org/apache/manifoldcf/crawlerui/IdleCleanupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/crawler-ui/src/main/java/org/apache/manifoldcf/crawlerui/IdleCleanupThread.java?rev=1640019&r1=1640018&r2=1640019&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/crawler-ui/src/main/java/org/apache/manifoldcf/crawlerui/IdleCleanupThread.java
(original)
+++ manifoldcf/branches/dev_1x/framework/crawler-ui/src/main/java/org/apache/manifoldcf/crawlerui/IdleCleanupThread.java
Sun Nov 16 17:48:07 2014
@@ -51,16 +51,12 @@ public class IdleCleanupThread extends T
     {
       // Create a thread context object.
       IThreadContext threadContext = ThreadContextFactory.make();
-      // Get the cache handle.
-      ICacheManager cacheManager = CacheManagerFactory.make(threadContext);
       
       IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
       IOutputConnectorPool outputConnectorPool = OutputConnectorPoolFactory.make(threadContext);
       ITransformationConnectorPool transformationConnectorPool = TransformationConnectorPoolFactory.make(threadContext);
       IAuthorityConnectorPool authorityConnectorPool = AuthorityConnectorPoolFactory.make(threadContext);
       IMappingConnectorPool mappingConnectorPool = MappingConnectorPoolFactory.make(threadContext);
-      
-      IThrottleGroups throttleGroups = ThrottleGroupsFactory.make(threadContext);
 
       // Loop
       while (true)
@@ -75,9 +71,8 @@ public class IdleCleanupThread extends T
           authorityConnectorPool.pollAllConnectors();
           mappingConnectorPool.pollAllConnectors();
           
-          throttleGroups.poll();
-          
-          cacheManager.expireObjects(System.currentTimeMillis());
+          // Poll all basic services
+          ManifoldCF.pollAll(threadContext);
           
           // Sleep for the retry interval.
           ManifoldCF.sleep(5000L);

Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/system/IdleCleanupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/system/IdleCleanupThread.java?rev=1640019&r1=1640018&r2=1640019&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/system/IdleCleanupThread.java
(original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/system/IdleCleanupThread.java
Sun Nov 16 17:48:07 2014
@@ -51,7 +51,6 @@ public class IdleCleanupThread extends T
     {
       // Create a thread context object.
       IThreadContext threadContext = ThreadContextFactory.make();
-      ICacheManager cacheManager = CacheManagerFactory.make(threadContext);
       IAuthorityConnectorPool authorityConnectorPool = AuthorityConnectorPoolFactory.make(threadContext);
       IMappingConnectorPool mappingConnectorPool = MappingConnectorPoolFactory.make(threadContext);
       
@@ -64,7 +63,8 @@ public class IdleCleanupThread extends T
           // Do the cleanup
           authorityConnectorPool.pollAllConnectors();
           mappingConnectorPool.pollAllConnectors();
-          cacheManager.expireObjects(System.currentTimeMillis());
+          // Poll all basic services
+          ManifoldCF.pollAll(threadContext);
           
           // Sleep for the retry interval.
           ManifoldCF.sleep(5000L);

Modified: manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java?rev=1640019&r1=1640018&r2=1640019&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java
(original)
+++ manifoldcf/branches/dev_1x/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/IdleCleanupThread.java
Sun Nov 16 17:48:07 2014
@@ -54,8 +54,6 @@ public class IdleCleanupThread extends T
     {
       // Create a thread context object.
       IThreadContext threadContext = ThreadContextFactory.make();
-      // Get the cache handle.
-      ICacheManager cacheManager = CacheManagerFactory.make(threadContext);
       
       IRepositoryConnectorPool repositoryConnectorPool = RepositoryConnectorPoolFactory.make(threadContext);
       
@@ -67,7 +65,7 @@ public class IdleCleanupThread extends T
         {
           // Do the cleanup
           repositoryConnectorPool.pollAllConnectors();
-          cacheManager.expireObjects(System.currentTimeMillis());
+          ManifoldCF.pollAll(threadContext);
           
           // Sleep for the retry interval.
           ManifoldCF.sleep(5000L);



Mime
View raw message