manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1204911 - in /incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs: JobManager.java JobQueue.java
Date Tue, 22 Nov 2011 10:49:58 GMT
Author: kwright
Date: Tue Nov 22 10:49:58 2011
New Revision: 1204911

URL: http://svn.apache.org/viewvc?rev=1204911&view=rev
Log:
First part of changes related to CONNECTORS-290.  This includes setting the docpriority to
null for all non-PENDING and non-PENDING_PURGATORY documents, as well as setting the docpriority
field to null when a job aborts.

Modified:
    incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
    incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java

Modified: incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1204911&r1=1204910&r2=1204911&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
(original)
+++ incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Tue Nov 22 10:49:58 2011
@@ -1470,6 +1470,10 @@ public class JobManager implements IJobM
 
     list.add(jobQueue.actionToString(JobQueue.ACTION_RESCAN));
 
+    // Per CONNECTORS-290, we need to be leaving priorities blank for jobs that aren't using
them,
+    // so this will be changed to not include jobs where the priorities have been bashed
to null.
+    // MHL
+        
     sb.append("EXISTS(SELECT 'x' FROM ").append(jobs.getTableName()).append(" t1 WHERE ")
       .append(database.buildConjunctionClause(list,new ClauseDescription[]{
         new MultiClause("t1."+jobs.statusField,new Object[]{
@@ -1477,7 +1481,6 @@ public class JobManager implements IJobM
           Jobs.statusToString(Jobs.STATUS_PAUSED),
           Jobs.statusToString(Jobs.STATUS_ACTIVEWAIT),
           Jobs.statusToString(Jobs.STATUS_PAUSEDWAIT),
-          Jobs.statusToString(Jobs.STATUS_ACTIVE),
           Jobs.statusToString(Jobs.STATUS_READYFORSTARTUP),
           Jobs.statusToString(Jobs.STATUS_STARTINGUP),
           Jobs.statusToString(Jobs.STATUS_ABORTINGSTARTINGUPFORRESTART),
@@ -6223,6 +6226,11 @@ public class JobManager implements IJobM
           if (confirmSet.getRowCount() > 0)
             continue;
 
+          // All the job's documents need to have their docpriority set to null, to clear
dead wood out of the docpriority index.
+          // See CONNECTORS-290.
+          // We do this BEFORE updating the job state.
+          jobQueue.clearDocPriorities(jobID);
+          
           int status = jobs.stringToStatus((String)row.getValue(jobs.statusField));
           IJobDescription jobDesc = jobs.load(jobID,true);
           abortJobs.add(jobDesc);
@@ -6240,6 +6248,7 @@ public class JobManager implements IJobM
           case Jobs.STATUS_ABORTINGFORRESTART:
             // Do the restart sequence!  Log the abort here; the startup thread will log
the start.
             jobs.startJob(jobID,null);
+            if (Logging.jobs.isDebugEnabled())
             {
               Logging.jobs.debug("Completed restart of job "+jobID);
             }

Modified: incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java?rev=1204911&r1=1204910&r2=1204911&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
(original)
+++ incubator/lcf/branches/CONNECTORS-290/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
Tue Nov 22 10:49:58 2011
@@ -547,11 +547,7 @@ public class JobQueue extends org.apache
     // Delete PENDING and ACTIVE entries
     HashMap map = new HashMap();
     map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
-    // Do not reset priorities.  This means, of course, that they may be out of date - but
they are probably more accurate in their current form
-    // than being set back to some arbitrary value.
-    // The alternative, which would be to reprioritize all the documents at this point, is
somewhat attractive, but let's see if we can get away
-    // without for now.
-    //map.put(docPriorityField,new Double(1.0));
+    // Do not reset priorities here!  They should all be blank at this point.
     map.put(checkTimeField,new Long(0L));
     map.put(checkActionField,actionToString(ACTION_RESCAN));
     map.put(failTimeField,null);
@@ -633,14 +629,31 @@ public class JobQueue extends org.apache
     noteModifications(0,1,0);
   }
 
+  /** Clear all document priorities for a job */
+  public void clearDocPriorities(Long jobID)
+    throws ManifoldCFException
+  {
+    HashMap map = new HashMap();
+    map.put(prioritySetField,null);
+    map.put(docPriorityField,null);
+    ArrayList list = new ArrayList();
+    String query = buildConjunctionClause(list,new ClauseDescription[]{
+      new UnitaryClause(jobIDField,jobID)});
+    performUpdate(map,"WHERE "+query,list,null);
+    noteModifications(0,1,0);
+  }
+  
   /** Set the "completed" status for a record.
   */
   public void updateCompletedRecord(Long recID, int currentStatus)
     throws ManifoldCFException
   {
+    HashMap map = new HashMap();
+    
     int newStatus;
     String actionFieldValue;
     Long checkTimeValue;
+    
     switch (currentStatus)
     {
     case STATUS_ACTIVE:
@@ -648,27 +661,26 @@ public class JobQueue extends org.apache
       newStatus = STATUS_COMPLETE;
       actionFieldValue = null;
       checkTimeValue = null;
+      // Remove document priority; we don't want to pollute the queue.  See CONNECTORS-290.
+      map.put(docPriorityField,null);
+      map.put(prioritySetField,null);
       break;
     case STATUS_ACTIVENEEDRESCAN:
     case STATUS_ACTIVENEEDRESCANPURGATORY:
       newStatus = STATUS_PENDINGPURGATORY;
       actionFieldValue = actionToString(ACTION_RESCAN);
       checkTimeValue = new Long(0L);
+      // Leave doc priority unchanged.
       break;
     default:
       throw new ManifoldCFException("Unexpected jobqueue status - record id "+recID.toString()+",
expecting active status, saw "+Integer.toString(currentStatus));
     }
 
-    HashMap map = new HashMap();
     map.put(statusField,statusToString(newStatus));
     map.put(checkTimeField,checkTimeValue);
     map.put(checkActionField,actionFieldValue);
     map.put(failTimeField,null);
     map.put(failCountField,null);
-    // Don't rejigger document priority, because it is hard to calculate and because what's
there is probably better
-    // than any arbitrary value I'd use.
-    //map.put(docPriorityField,new Double(1.0));
-    //map.put(prioritySetField,new Long(0L));
     ArrayList list = new ArrayList();
     String query = buildConjunctionClause(list,new ClauseDescription[]{
       new UnitaryClause(idField,recID)});
@@ -705,6 +717,7 @@ public class JobQueue extends org.apache
   }
 
   /** Set the status on a record, including check time and priority.
+  * The status set MUST be a PENDING or PENDINGPURGATORY status.
   *@param id is the job queue id.
   *@param status is the desired status
   *@param checkTime is the check time.



Mime
View raw message