incubator-connectors-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1204992 - in /incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system: JobResetThread.java ManifoldCF.java
Date Tue, 22 Nov 2011 13:48:31 GMT
Author: kwright
Date: Tue Nov 22 13:48:31 2011
New Revision: 1204992

URL: http://svn.apache.org/viewvc?rev=1204992&view=rev
Log:
Move logic for reprioritizing documents to a more generic location.

Modified:
    incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobResetThread.java
    incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java

Modified: incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobResetThread.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobResetThread.java?rev=1204992&r1=1204991&r2=1204992&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobResetThread.java
(original)
+++ incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobResetThread.java
Tue Nov 22 13:48:31 2011
@@ -92,43 +92,8 @@ public class JobResetThread extends Thre
           {
             Logging.threads.debug("Job reset thread reprioritizing documents...");
 
-            // Reset the queue tracker
-            queueTracker.beginReset();
-            // Perform the reprioritization, for all active documents in active jobs.  During
this time,
-            // it is safe to have other threads assign new priorities to documents, but it
is NOT safe
-            // for other threads to attempt to change the minimum priority level.  The queuetracker
object
-            // will therefore block that from occurring, until the reset is complete.
-            try
-            {
-              // Reprioritize all documents in the jobqueue, 1000 at a time
-
-              HashMap connectionMap = new HashMap();
-              HashMap jobDescriptionMap = new HashMap();
-
-              // Do the 'not yet processed' documents only.  Documents that are queued for
reprocessing will be assigned
-              // new priorities.  Already processed documents won't.  This guarantees that
our bins are appropriate for current thread
-              // activity.
-              // In order for this to be the correct functionality, ALL reseeding and requeuing
operations MUST reset the associated document
-              // priorities.
-              while (true)
-              {
-                long startTime = System.currentTimeMillis();
-
-                DocumentDescription[] docs = jobManager.getNextNotYetProcessedReprioritizationDocuments(currentTime,
10000);
-                if (docs.length == 0)
-                  break;
-
-                // Calculate new priorities for all these documents
-                ManifoldCF.writeDocumentPriorities(threadContext,connectionManager,jobManager,docs,connectionMap,jobDescriptionMap,queueTracker,currentTime);
-
-                Logging.threads.debug("Reprioritized "+Integer.toString(docs.length)+" not-yet-processed
documents in "+new Long(System.currentTimeMillis()-startTime)+" ms");
-              }
-            }
-            finally
-            {
-              queueTracker.endReset();
-            }
-
+            ManifoldCF.resetAllDocumentPriorities(threadContext,queueTracker,currentTime);
+            
             Logging.threads.debug("Job reset thread done reprioritizing documents.");
 
           }

Modified: incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java?rev=1204992&r1=1204991&r2=1204992&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
(original)
+++ incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
Tue Nov 22 13:48:31 2011
@@ -951,6 +951,55 @@ public class ManifoldCF extends org.apac
     return connector.getBinNames(documentIdentifier);
   }
 
+  /** Reset all (active) document priorities.  This operation may occur due to various externally-triggered
+  * events, such a job abort, pause, resume, wait, or unwait.
+  */
+  public static void resetAllDocumentPriorities(IThreadContext threadContext, QueueTracker
queueTracker, long currentTime)
+    throws ManifoldCFException
+  {
+    IJobManager jobManager = JobManagerFactory.make(threadContext);
+    IRepositoryConnectionManager connectionManager = RepositoryConnectionManagerFactory.make(threadContext);
+    
+    // Reset the queue tracker
+    queueTracker.beginReset();
+    // Perform the reprioritization, for all active documents in active jobs.  During this
time,
+    // it is safe to have other threads assign new priorities to documents, but it is NOT
safe
+    // for other threads to attempt to change the minimum priority level.  The queuetracker
object
+    // will therefore block that from occurring, until the reset is complete.
+    try
+    {
+      // Reprioritize all documents in the jobqueue, 1000 at a time
+
+      HashMap connectionMap = new HashMap();
+      HashMap jobDescriptionMap = new HashMap();
+
+      // Do the 'not yet processed' documents only.  Documents that are queued for reprocessing
will be assigned
+      // new priorities.  Already processed documents won't.  This guarantees that our bins
are appropriate for current thread
+      // activity.
+      // In order for this to be the correct functionality, ALL reseeding and requeuing operations
MUST reset the associated document
+      // priorities.
+      while (true)
+      {
+        long startTime = System.currentTimeMillis();
+
+        DocumentDescription[] docs = jobManager.getNextNotYetProcessedReprioritizationDocuments(currentTime,
10000);
+        if (docs.length == 0)
+          break;
+
+        // Calculate new priorities for all these documents
+        writeDocumentPriorities(threadContext,connectionManager,jobManager,docs,connectionMap,jobDescriptionMap,queueTracker,currentTime);
+
+        Logging.threads.debug("Reprioritized "+Integer.toString(docs.length)+" not-yet-processed
documents in "+new Long(System.currentTimeMillis()-startTime)+" ms");
+      }
+    }
+    finally
+    {
+      queueTracker.endReset();
+    }
+  }
+  
+  /** Write a set of document priorities, based on the current queue tracker.
+  */
   public static void writeDocumentPriorities(IThreadContext threadContext, IRepositoryConnectionManager
mgr, IJobManager jobManager, DocumentDescription[] descs, HashMap connectionMap, HashMap jobDescriptionMap,
QueueTracker queueTracker, long currentTime)
     throws ManifoldCFException
   {



Mime
View raw message