manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1457305 [2/3] - in /manifoldcf/trunk: ./ connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/ framework/crawler-ui/src/main/webapp/ framework/pull-agent/src/main/java/org/apache/manifoldcf/craw...
Date Sat, 16 Mar 2013 21:03:09 GMT
Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java Sat Mar 16 21:03:08 2013
@@ -98,7 +98,7 @@ public class JobManager implements IJobM
     throws java.io.IOException, ManifoldCFException
   {
     // Write a version indicator
-    ManifoldCF.writeDword(os,2);
+    ManifoldCF.writeDword(os,3);
     // Get the job list
     IJobDescription[] list = getAllJobs();
     // Write the number of authorities
@@ -136,6 +136,7 @@ public class JobManager implements IJobM
         writeEnumeratedValues(os,sr.getMinutesOfHour());
         ManifoldCF.writeString(os,sr.getTimezone());
         ManifoldCF.writeLong(os,sr.getDuration());
+        ManifoldCF.writeByte(os,sr.getRequestMinimum()?1:0);
       }
 
       // Write hop count filters
@@ -174,7 +175,7 @@ public class JobManager implements IJobM
     throws java.io.IOException, ManifoldCFException
   {
     int version = ManifoldCF.readDword(is);
-    if (version != 2)
+    if (version != 2 && version != 3)
       throw new java.io.IOException("Unknown job configuration version: "+Integer.toString(version));
     int count = ManifoldCF.readDword(is);
     int i = 0;
@@ -208,9 +209,14 @@ public class JobManager implements IJobM
         EnumeratedValues minutesOfHour = readEnumeratedValues(is);
         String timezone = ManifoldCF.readString(is);
         Long duration = ManifoldCF.readLong(is);
+        boolean requestMinimum;
+        if (version >= 3)
+          requestMinimum = (ManifoldCF.readByte(is) != 0);
+        else
+          requestMinimum = false;
 
         ScheduleRecord sr = new ScheduleRecord(dayOfWeek, monthOfYear, dayOfMonth, year,
-          hourOfDay, minutesOfHour, timezone, duration);
+          hourOfDay, minutesOfHour, timezone, duration, requestMinimum);
         job.addScheduleRecord(sr);
         j++;
       }
@@ -1343,11 +1349,12 @@ public class JobManager implements IJobM
     sb.append("SELECT t0.").append(jobQueue.docHashField).append(" FROM ").append(jobQueue.getTableName()).append(" t0 WHERE ")
       .append(database.buildConjunctionClause(list,new ClauseDescription[]{
         new MultiClause("t0."+jobQueue.docHashField,docList)})).append(" AND ")
-      .append("t0.").append(jobQueue.statusField).append(" IN (?,?,?,?) AND ");
+      .append("t0.").append(jobQueue.statusField).append(" IN (?,?,?,?,?) AND ");
 
     list.add(jobQueue.statusToString(jobQueue.STATUS_PURGATORY));
     list.add(jobQueue.statusToString(jobQueue.STATUS_PENDINGPURGATORY));
     list.add(jobQueue.statusToString(jobQueue.STATUS_COMPLETE));
+    list.add(jobQueue.statusToString(jobQueue.STATUS_UNCHANGED));
     list.add(jobQueue.statusToString(jobQueue.STATUS_ELIGIBLEFORDELETE));
     
     sb.append("EXISTS(SELECT 'x' FROM ").append(jobs.getTableName()).append(" t1 WHERE ")
@@ -1424,6 +1431,7 @@ public class JobManager implements IJobM
     sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{
       new MultiClause(jobQueue.statusField,new Object[]{
         jobQueue.statusToString(JobQueue.STATUS_COMPLETE),
+        jobQueue.statusToString(JobQueue.STATUS_UNCHANGED),
         jobQueue.statusToString(JobQueue.STATUS_PURGATORY)}),
       new UnitaryClause(jobQueue.prioritySetField,"<",new Long(currentTime))})).append(" ");
       
@@ -1490,6 +1498,7 @@ public class JobManager implements IJobM
       .append(database.buildConjunctionClause(list,new ClauseDescription[]{
         new MultiClause("t1."+jobs.statusField,new Object[]{
           Jobs.statusToString(Jobs.STATUS_STARTINGUP),
+          Jobs.statusToString(Jobs.STATUS_STARTINGUPMINIMAL),
           Jobs.statusToString(Jobs.STATUS_ACTIVE),
           Jobs.statusToString(Jobs.STATUS_ACTIVESEEDING),
           Jobs.statusToString(Jobs.STATUS_ACTIVE_UNINSTALLED),
@@ -4745,13 +4754,14 @@ public class JobManager implements IJobM
 
           // We go through *all* the schedule records.  The one that matches that has the latest
           // end time is the one we take.
-          int l = 0;
           Long matchTime = null;
           Long duration = null;
-          while (l < thisSchedule.length)
+          boolean requestMinimum = false;
+          
+          for (int l = 0; l < thisSchedule.length; l++)
           {
             long trialStartInterval = startInterval;
-            ScheduleRecord sr = thisSchedule[l++];
+            ScheduleRecord sr = thisSchedule[l];
             Long thisDuration = sr.getDuration();
             if (startMethod == IJobDescription.START_WINDOWINSIDE &&
               thisDuration != null)
@@ -4787,10 +4797,11 @@ public class JobManager implements IJobM
 
             if (matchTime == null || thisDuration == null ||
               (duration != null && thisMatchTime.longValue() + thisDuration.longValue() >
-            matchTime.longValue() + duration.longValue()))
+                matchTime.longValue() + duration.longValue()))
             {
               matchTime = thisMatchTime;
               duration = thisDuration;
+              requestMinimum = sr.getRequestMinimum();
             }
           }
 
@@ -4823,7 +4834,7 @@ public class JobManager implements IJobM
             // If job was formerly "inactive", do the full startup.
             // Start this job!  but with no end time.
             // This does not get logged because the startup thread does the logging.
-            jobs.startJob(jobID,windowEnd);
+            jobs.startJob(jobID,windowEnd,requestMinimum);
             jobQueue.clearFailTimes(jobID);
             if (Logging.jobs.isDebugEnabled())
             {
@@ -5256,6 +5267,18 @@ public class JobManager implements IJobM
   public void manualStart(Long jobID)
     throws ManifoldCFException
   {
+    manualStart(jobID,false);
+  }
+  
+  /** Manually start a job.  The specified job will be run REGARDLESS of the timed windows, and
+  * will not cease until complete.  If the job is already running, this operation will assure that
+  * the job does not pause when its window ends.  The job can be manually paused, or manually aborted.
+  *@param jobID is the ID of the job to start.
+  *@param requestMinimum is true if a minimal job run is requested.
+  */
+  public void manualStart(Long jobID, boolean requestMinimum)
+    throws ManifoldCFException
+  {
     database.beginTransaction();
     try
     {
@@ -5284,8 +5307,9 @@ public class JobManager implements IJobM
         Logging.jobs.debug("Manually starting job "+jobID);
       }
       // Start this job!  but with no end time.
-      jobs.startJob(jobID,null);
+      jobs.startJob(jobID,null,requestMinimum);
       jobQueue.clearFailTimes(jobID);
+      
       if (Logging.jobs.isDebugEnabled())
       {
         Logging.jobs.debug("Manual job start signal for job "+jobID+" successfully sent");
@@ -5355,12 +5379,71 @@ public class JobManager implements IJobM
     jobQueue.prepareDeleteScan(jobID);
   }
   
+  /** Prepare a job to be run.
+  * This method is called regardless of the details of the job; what differs is only the flags that are passed in.
+  * The code inside will determine the appropriate procedures.
+  * (This method replaces prepareFullScan() and prepareIncrementalScan(). )
+  *@param jobID is the job id.
+  *@param legalLinkTypes are the link types allowed for the job.
+  *@param hopcountMethod describes how to handle deletions for hopcount purposes.
+  *@param connectorModel is the model used by the connector for the job.
+  *@param continuousJob is true if the job is a continuous one.
+  *@param fromBeginningOfTime is true if the job is running starting from time 0.
+  *@param requestMinimum is true if the minimal amount of work is requested for the job run.
+  */
+  public void prepareJobScan(Long jobID, String[] legalLinkTypes, int hopcountMethod,
+    int connectorModel, boolean continuousJob, boolean fromBeginningOfTime,
+    boolean requestMinimum)
+    throws ManifoldCFException
+  {
+    // (1) If the connector has MODEL_ADD_CHANGE_DELETE, then
+    // we let the connector run the show; there's no purge phase, and therefore the
+    // documents are left in a COMPLETED state if they don't show up in the list
+    // of seeds that require the attention of the connector.
+    //
+    // (2) If the connector has MODEL_ALL, then it's a full crawl no matter what, so
+    // we do a full scan initialization.
+    //
+    // (3) If the connector has some other model, we look at the start time.  A start
+    // time of 0 implies a full scan, while any other start time implies an incremental
+    // scan.
+
+    // Complete connector model is told everything, so no delete phase.
+    if (connectorModel == IRepositoryConnector.MODEL_ADD_CHANGE_DELETE)
+      return;
+    
+    // If the connector model is complete via chaining, then we just need to make
+    // sure discovery works to queue the changes.
+    if (connectorModel == IRepositoryConnector.MODEL_CHAINED_ADD_CHANGE_DELETE)
+    {
+      jobQueue.preparePartialScan(jobID);
+      return;
+    }
+    
+    // Similarly, minimal crawl attempts no delete phase unless the connector explicitly forbids it, or unless
+    // the job criteria have changed.
+    if (requestMinimum && connectorModel != IRepositoryConnector.MODEL_ALL && !fromBeginningOfTime)
+    {
+      // If it is a chained model, do the partial prep.
+      if (connectorModel == IRepositoryConnector.MODEL_CHAINED_ADD ||
+        connectorModel == IRepositoryConnector.MODEL_CHAINED_ADD_CHANGE)
+        jobQueue.preparePartialScan(jobID);
+      return;
+    }
+    
+    if (!continuousJob && connectorModel != IRepositoryConnector.MODEL_PARTIAL &&
+      (connectorModel == IRepositoryConnector.MODEL_ALL || fromBeginningOfTime))
+      prepareFullScan(jobID,legalLinkTypes,hopcountMethod);
+    else
+      jobQueue.prepareIncrementalScan(jobID);
+  }
+
   /** Prepare for a full scan.
   *@param jobID is the job id.
   *@param legalLinkTypes are the link types allowed for the job.
   *@param hopcountMethod describes how to handle deletions for hopcount purposes.
   */
-  public void prepareFullScan(Long jobID, String[] legalLinkTypes, int hopcountMethod)
+  protected void prepareFullScan(Long jobID, String[] legalLinkTypes, int hopcountMethod)
     throws ManifoldCFException
   {
     while (true)
@@ -5370,7 +5453,7 @@ public class JobManager implements IJobM
       database.beginTransaction(database.TRANSACTION_SERIALIZED);
       try
       {
-        // Delete all documents that match a given criteria
+        // Delete the documents we have never fetched, including any hopcount records we've calculated.
         if (legalLinkTypes.length > 0)
         {
           ArrayList list = new ArrayList();
@@ -5413,17 +5496,6 @@ public class JobManager implements IJobM
     }
   }
 
-  /** Prepare for an incremental scan.
-  *@param jobID is the job id.
-  *@param legalLinkTypes are the link types allowed for the job.
-  *@param hopcountMethod describes how to handle deletions for hopcount purposes.
-  */
-  public void prepareIncrementalScan(Long jobID, String[] legalLinkTypes, int hopcountMethod)
-    throws ManifoldCFException
-  {
-    jobQueue.prepareIncrementalScan(jobID);
-  }
-
   /** Manually abort a running job.  The job will be permanently stopped, and will not run again until
   * automatically started based on schedule, or manually started.
   *@param jobID is the job to abort.
@@ -5479,8 +5551,9 @@ public class JobManager implements IJobM
   /** Manually restart a running job.  The job will be stopped and restarted.  Any schedule affinity will be lost,
   * until the job finishes on its own.
   *@param jobID is the job to abort.
+  *@param requestMinimum is true if a minimal job run is requested.
   */
-  public void manualAbortRestart(Long jobID)
+  public void manualAbortRestart(Long jobID, boolean requestMinimum)
     throws ManifoldCFException
   {
     if (Logging.jobs.isDebugEnabled())
@@ -5493,7 +5566,7 @@ public class JobManager implements IJobM
       database.beginTransaction();
       try
       {
-        jobs.abortRestartJob(jobID);
+        jobs.abortRestartJob(jobID,requestMinimum);
         database.performCommit();
         break;
       }
@@ -5526,6 +5599,16 @@ public class JobManager implements IJobM
     }
   }
 
+  /** Manually restart a running job.  The job will be stopped and restarted.  Any schedule affinity will be lost,
+  * until the job finishes on its own.
+  *@param jobID is the job to abort.
+  */
+  public void manualAbortRestart(Long jobID)
+    throws ManifoldCFException
+  {
+    manualAbortRestart(jobID,false);
+  }
+
   /** Abort a running job due to a fatal error condition.
   *@param jobID is the job to abort.
   *@param errorText is the error text.
@@ -5687,7 +5770,7 @@ public class JobManager implements IJobM
   *@return jobs that are active and are running in adaptive mode.  These will be seeded
   * based on what the connector says should be added to the queue.
   */
-  public JobStartRecord[] getJobsReadyForSeeding(long currentTime)
+  public JobSeedingRecord[] getJobsReadyForSeeding(long currentTime)
     throws ManifoldCFException
   {
     while (true)
@@ -5715,7 +5798,7 @@ public class JobManager implements IJobM
         
         IResultSet set = database.performQuery(sb.toString(),list,null,null);
         // Update them all
-        JobStartRecord[] rval = new JobStartRecord[set.getRowCount()];
+        JobSeedingRecord[] rval = new JobSeedingRecord[set.getRowCount()];
         int i = 0;
         while (i < rval.length)
         {
@@ -5741,7 +5824,7 @@ public class JobManager implements IJobM
             Logging.jobs.debug("Marked job "+jobID+" for seeding");
           }
 
-          rval[i] = new JobStartRecord(jobID,synchTime);
+          rval[i] = new JobSeedingRecord(jobID,synchTime);
           i++;
         }
         database.performCommit();
@@ -5775,7 +5858,7 @@ public class JobManager implements IJobM
   /** Get the list of jobs that are ready for deletion.
   *@return jobs that were in the "readyfordelete" state.
   */
-  public JobStartRecord[] getJobsReadyForDelete()
+  public JobDeleteRecord[] getJobsReadyForDelete()
     throws ManifoldCFException
   {
     while (true)
@@ -5795,7 +5878,7 @@ public class JobManager implements IJobM
             
         IResultSet set = database.performQuery(sb.toString(),list,null,null);
         // Update them all
-        JobStartRecord[] rval = new JobStartRecord[set.getRowCount()];
+        JobDeleteRecord[] rval = new JobDeleteRecord[set.getRowCount()];
         int i = 0;
         while (i < rval.length)
         {
@@ -5809,7 +5892,7 @@ public class JobManager implements IJobM
             Logging.jobs.debug("Marked job "+jobID+" for delete startup");
           }
 
-          rval[i] = new JobStartRecord(jobID,0L);
+          rval[i] = new JobDeleteRecord(jobID);
           i++;
         }
         database.performCommit();
@@ -5857,10 +5940,13 @@ public class JobManager implements IJobM
         ArrayList list = new ArrayList();
         
         sb.append(jobs.idField).append(",")
-          .append(jobs.lastCheckTimeField)
+          .append(jobs.lastCheckTimeField).append(",")
+          .append(jobs.statusField)
           .append(" FROM ").append(jobs.getTableName()).append(" WHERE ")
           .append(database.buildConjunctionClause(list,new ClauseDescription[]{
-            new UnitaryClause(jobs.statusField,jobs.statusToString(jobs.STATUS_READYFORSTARTUP))}))
+            new MultiClause(jobs.statusField,new Object[]{
+              jobs.statusToString(jobs.STATUS_READYFORSTARTUP),
+              jobs.statusToString(jobs.STATUS_READYFORSTARTUPMINIMAL)})}))
           .append(" FOR UPDATE");
             
         IResultSet set = database.performQuery(sb.toString(),list,null,null);
@@ -5872,18 +5958,22 @@ public class JobManager implements IJobM
           IResultRow row = set.getRow(i);
           Long jobID = (Long)row.getValue(jobs.idField);
           Long x = (Long)row.getValue(jobs.lastCheckTimeField);
+          int status = jobs.stringToStatus((String)row.getValue(jobs.statusField));
+
+          boolean requestMinimum = (status == jobs.STATUS_READYFORSTARTUPMINIMAL);
+          
           long synchTime = 0;
           if (x != null)
             synchTime = x.longValue();
 
           // Mark status of job as "starting"
-          jobs.writeStatus(jobID,jobs.STATUS_STARTINGUP);
+          jobs.writeStatus(jobID,requestMinimum?jobs.STATUS_STARTINGUPMINIMAL:jobs.STATUS_STARTINGUP);
           if (Logging.jobs.isDebugEnabled())
           {
             Logging.jobs.debug("Marked job "+jobID+" for startup");
           }
 
-          rval[i] = new JobStartRecord(jobID,synchTime);
+          rval[i] = new JobStartRecord(jobID,synchTime,requestMinimum);
           i++;
         }
         database.performCommit();
@@ -6151,7 +6241,15 @@ public class JobManager implements IJobM
           // Set the state of the job back to "ReadyForStartup"
           jobs.writeStatus(jobID,jobs.STATUS_READYFORSTARTUP);
           break;
+        case Jobs.STATUS_STARTINGUPMINIMAL:
+          if (Logging.jobs.isDebugEnabled())
+            Logging.jobs.debug("Setting job "+jobID+" back to 'ReadyForStartupMinimal' state");
+
+          // Set the state of the job back to "ReadyForStartupMinimal"
+          jobs.writeStatus(jobID,jobs.STATUS_READYFORSTARTUPMINIMAL);
+          break;
         case Jobs.STATUS_ABORTINGSTARTINGUP:
+        case Jobs.STATUS_ABORTINGSTARTINGUPMINIMAL:
           if (Logging.jobs.isDebugEnabled())
             Logging.jobs.debug("Setting job "+jobID+" to 'Aborting' state");
           jobs.writeStatus(jobID,jobs.STATUS_ABORTING);
@@ -6161,10 +6259,17 @@ public class JobManager implements IJobM
             Logging.jobs.debug("Setting job "+jobID+" to 'AbortingForRestart' state");
           jobs.writeStatus(jobID,jobs.STATUS_ABORTINGFORRESTART);
           break;
+        case Jobs.STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL:
+          if (Logging.jobs.isDebugEnabled())
+            Logging.jobs.debug("Setting job "+jobID+" to 'AbortingForRestartMinimal' state");
+          jobs.writeStatus(jobID,jobs.STATUS_ABORTINGFORRESTARTMINIMAL);
+          break;
 
         case Jobs.STATUS_READYFORSTARTUP:
+        case Jobs.STATUS_READYFORSTARTUPMINIMAL:
         case Jobs.STATUS_ABORTING:
         case Jobs.STATUS_ABORTINGFORRESTART:
+        case Jobs.STATUS_ABORTINGFORRESTARTMINIMAL:
           // ok
           break;
         default:
@@ -6291,8 +6396,17 @@ public class JobManager implements IJobM
           jobs.writeStatus(jobID,jobs.STATUS_ABORTINGFORRESTART);
           break;
 
+        case Jobs.STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL:
+          if (Logging.jobs.isDebugEnabled())
+            Logging.jobs.debug("Setting job "+jobID+" back to 'AbortingForRestartMinimal' state");
+
+          // Set the state of the job back to "Active"
+          jobs.writeStatus(jobID,jobs.STATUS_ABORTINGFORRESTARTMINIMAL);
+          break;
+
         case Jobs.STATUS_ABORTING:
         case Jobs.STATUS_ABORTINGFORRESTART:
+        case Jobs.STATUS_ABORTINGFORRESTARTMINIMAL:
         case Jobs.STATUS_ACTIVE:
         case Jobs.STATUS_ACTIVE_UNINSTALLED:
         case Jobs.STATUS_ACTIVE_NOOUTPUT:
@@ -6544,7 +6658,7 @@ public class JobManager implements IJobM
   /** Find the list of jobs that need to have their connectors notified of job completion.
   *@return the ID's of jobs that need their output connectors notified in order to become inactive.
   */
-  public JobStartRecord[] getJobsReadyForInactivity()
+  public JobNotifyRecord[] getJobsReadyForInactivity()
     throws ManifoldCFException
   {
     while (true)
@@ -6564,7 +6678,7 @@ public class JobManager implements IJobM
             
         IResultSet set = database.performQuery(sb.toString(),list,null,null);
         // Return them all
-        JobStartRecord[] rval = new JobStartRecord[set.getRowCount()];
+        JobNotifyRecord[] rval = new JobNotifyRecord[set.getRowCount()];
         int i = 0;
         while (i < rval.length)
         {
@@ -6576,7 +6690,7 @@ public class JobManager implements IJobM
           {
             Logging.jobs.debug("Found job "+jobID+" in need of notification");
           }
-          rval[i++] = new JobStartRecord(jobID,0L);
+          rval[i++] = new JobNotifyRecord(jobID);
         }
         database.performCommit();
         return rval;
@@ -6672,6 +6786,7 @@ public class JobManager implements IJobM
         new MultiClause(jobs.statusField,new Object[]{
           jobs.statusToString(jobs.STATUS_ABORTING),
           jobs.statusToString(jobs.STATUS_ABORTINGFORRESTART),
+          jobs.statusToString(jobs.STATUS_ABORTINGFORRESTARTMINIMAL),
           jobs.statusToString(jobs.STATUS_PAUSING),
           jobs.statusToString(jobs.STATUS_PAUSINGSEEDING),
           jobs.statusToString(jobs.STATUS_ACTIVEWAITING),
@@ -7019,6 +7134,7 @@ public class JobManager implements IJobM
         .append(database.buildConjunctionClause(list,new ClauseDescription[]{
           new MultiClause(JobQueue.statusField,new Object[]{
             JobQueue.statusToString(JobQueue.STATUS_COMPLETE),
+            JobQueue.statusToString(JobQueue.STATUS_UNCHANGED),
             JobQueue.statusToString(JobQueue.STATUS_PURGATORY),
             JobQueue.statusToString(JobQueue.STATUS_ACTIVEPURGATORY),
             JobQueue.statusToString(JobQueue.STATUS_ACTIVENEEDRESCANPURGATORY),
@@ -7131,11 +7247,15 @@ public class JobManager implements IJobM
       case Jobs.STATUS_ABORTING:
       case Jobs.STATUS_ABORTINGSEEDING:
       case Jobs.STATUS_ABORTINGSTARTINGUP:
+      case Jobs.STATUS_ABORTINGSTARTINGUPMINIMAL:
         rstatus = JobStatus.JOBSTATUS_ABORTING;
         break;
       case Jobs.STATUS_ABORTINGFORRESTART:
+      case Jobs.STATUS_ABORTINGFORRESTARTMINIMAL:
       case Jobs.STATUS_ABORTINGFORRESTARTSEEDING:
+      case Jobs.STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL:
       case Jobs.STATUS_ABORTINGSTARTINGUPFORRESTART:
+      case Jobs.STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL:
         rstatus = JobStatus.JOBSTATUS_RESTARTING;
         break;
       case Jobs.STATUS_PAUSING:
@@ -7163,7 +7283,9 @@ public class JobManager implements IJobM
         rstatus = JobStatus.JOBSTATUS_PAUSED;
         break;
       case Jobs.STATUS_STARTINGUP:
+      case Jobs.STATUS_STARTINGUPMINIMAL:
       case Jobs.STATUS_READYFORSTARTUP:
+      case Jobs.STATUS_READYFORSTARTUPMINIMAL:
         rstatus = JobStatus.JOBSTATUS_STARTING;
         break;
       case Jobs.STATUS_DELETESTARTINGUP:
@@ -7221,6 +7343,7 @@ public class JobManager implements IJobM
       .append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Processed'")
       .append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Processed'")
       .append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Processed'")
+      .append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Processed'")
       .append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Being removed'")
       .append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Being removed'")
       .append(" WHEN ").append("t0.").append(jobQueue.statusField).append("=? THEN 'Being removed'")
@@ -7229,7 +7352,7 @@ public class JobManager implements IJobM
       .append(" END AS state,")
       .append("CASE")
       .append(" WHEN ")
-      .append("t0.").append(jobQueue.statusField).append(" IN (?,?)")
+      .append("t0.").append(jobQueue.statusField).append(" IN (?,?,?)")
       .append(" THEN 'Inactive'")
       .append(" WHEN ")
       .append("t0.").append(jobQueue.statusField).append(" IN (?,?)")
@@ -7289,6 +7412,7 @@ public class JobManager implements IJobM
     list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY));
     list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY));
     list.add(jobQueue.statusToString(jobQueue.STATUS_COMPLETE));
+    list.add(jobQueue.statusToString(jobQueue.STATUS_UNCHANGED));
     list.add(jobQueue.statusToString(jobQueue.STATUS_PURGATORY));
     list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED));
     list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED));
@@ -7296,6 +7420,7 @@ public class JobManager implements IJobM
     list.add(jobQueue.statusToString(jobQueue.STATUS_HOPCOUNTREMOVED));
     
     list.add(jobQueue.statusToString(jobQueue.STATUS_COMPLETE));
+    list.add(jobQueue.statusToString(jobQueue.STATUS_UNCHANGED));
     list.add(jobQueue.statusToString(jobQueue.STATUS_PURGATORY));
     
     list.add(jobQueue.statusToString(jobQueue.STATUS_PENDING));
@@ -7378,7 +7503,7 @@ public class JobManager implements IJobM
     sb.append(" AS idbucket,")
       .append("CASE")
       .append(" WHEN ")
-      .append(jobQueue.statusField).append(" IN (?,?)")
+      .append(jobQueue.statusField).append(" IN (?,?,?)")
       .append(" THEN 1 ELSE 0")
       .append(" END")
       .append(" AS inactive,")
@@ -7450,6 +7575,7 @@ public class JobManager implements IJobM
     sb.append(" FROM ").append(jobQueue.getTableName());
     
     list.add(jobQueue.statusToString(jobQueue.STATUS_COMPLETE));
+    list.add(jobQueue.statusToString(jobQueue.STATUS_UNCHANGED));
     list.add(jobQueue.statusToString(jobQueue.STATUS_PURGATORY));
     
     list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVE));
@@ -7576,6 +7702,7 @@ public class JobManager implements IJobM
             jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED),
             jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED),
             jobQueue.statusToString(jobQueue.STATUS_COMPLETE),
+            jobQueue.statusToString(jobQueue.STATUS_UNCHANGED),
             jobQueue.statusToString(jobQueue.STATUS_PURGATORY)})}));
         break;
       case DOCSTATE_OUTOFSCOPE:
@@ -7602,6 +7729,7 @@ public class JobManager implements IJobM
         sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{
           new MultiClause(fieldPrefix+jobQueue.statusField,new Object[]{
             jobQueue.statusToString(jobQueue.STATUS_COMPLETE),
+            jobQueue.statusToString(jobQueue.STATUS_UNCHANGED),
             jobQueue.statusToString(jobQueue.STATUS_PURGATORY)})}));
         break;
       case DOCSTATUS_PROCESSING:

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java Sat Mar 16 21:03:08 2013
@@ -60,15 +60,17 @@ public class JobQueue extends org.apache
   public final static int STATUS_PENDING = 0;
   public final static int STATUS_ACTIVE = 1;
   public final static int STATUS_COMPLETE = 2;
-  public final static int STATUS_PENDINGPURGATORY = 3;
-  public final static int STATUS_ACTIVEPURGATORY = 4;
-  public final static int STATUS_PURGATORY = 5;
-  public final static int STATUS_BEINGDELETED = 6;
-  public final static int STATUS_ACTIVENEEDRESCAN = 7;
-  public final static int STATUS_ACTIVENEEDRESCANPURGATORY = 8;
-  public final static int STATUS_BEINGCLEANED = 9;
-  public final static int STATUS_ELIGIBLEFORDELETE = 10;
-  public final static int STATUS_HOPCOUNTREMOVED = 11;
+  public final static int STATUS_UNCHANGED = 3;
+  public final static int STATUS_PENDINGPURGATORY = 4;
+  public final static int STATUS_ACTIVEPURGATORY = 5;
+  public final static int STATUS_PURGATORY = 6;
+  public final static int STATUS_BEINGDELETED = 7;
+  public final static int STATUS_ACTIVENEEDRESCAN = 8;
+  public final static int STATUS_ACTIVENEEDRESCANPURGATORY = 9;
+  public final static int STATUS_BEINGCLEANED = 10;
+  public final static int STATUS_ELIGIBLEFORDELETE = 11;
+  public final static int STATUS_HOPCOUNTREMOVED = 12;
+  
   // Action values
   public final static int ACTION_RESCAN = 0;
   public final static int ACTION_REMOVE = 1;
@@ -121,6 +123,7 @@ public class JobQueue extends org.apache
     statusMap.put("P",new Integer(STATUS_PENDING));
     statusMap.put("A",new Integer(STATUS_ACTIVE));
     statusMap.put("C",new Integer(STATUS_COMPLETE));
+    statusMap.put("U",new Integer(STATUS_UNCHANGED));
     statusMap.put("G",new Integer(STATUS_PENDINGPURGATORY));
     statusMap.put("F",new Integer(STATUS_ACTIVEPURGATORY));
     statusMap.put("Z",new Integer(STATUS_PURGATORY));
@@ -533,6 +536,7 @@ public class JobQueue extends org.apache
       new MultiClause(statusField,new Object[]{
         statusToString(STATUS_PENDINGPURGATORY),
         statusToString(STATUS_COMPLETE),
+        statusToString(STATUS_UNCHANGED),
         statusToString(STATUS_PURGATORY)})});
     performUpdate(map,"WHERE "+query,list,null);
 
@@ -583,6 +587,7 @@ public class JobQueue extends org.apache
       new UnitaryClause(jobIDField,jobID),
       new MultiClause(statusField,new Object[]{  
         statusToString(STATUS_PENDINGPURGATORY),
+        statusToString(STATUS_UNCHANGED),
         statusToString(STATUS_COMPLETE)})});
     performUpdate(map,"WHERE "+query,list,null);
 
@@ -592,6 +597,34 @@ public class JobQueue extends org.apache
     unconditionallyAnalyzeTables();
   }
 
+  /** Prepare for a "partial" job.  This is called ONLY when the job is inactive.
+  *
+  * This method maps all COMPLETE entries to UNCHANGED.  The purpose is to
+  * allow discovery to find the documents that need to be processed.  If they were
+  * marked as COMPLETE that would stop them from being queued.
+  *@param jobID is the job identifier.
+  */
+  public void preparePartialScan(Long jobID)
+    throws ManifoldCFException
+  {
+    // Map COMPLETE to UNCHANGED.
+    HashMap map = new HashMap();
+    map.put(statusField,statusToString(STATUS_UNCHANGED));
+    // Do not reset priorities here!  They should all be blank at this point.
+    map.put(checkTimeField,new Long(0L));
+    map.put(checkActionField,actionToString(ACTION_RESCAN));
+    map.put(failTimeField,null);
+    map.put(failCountField,null);
+    ArrayList list = new ArrayList();
+    String query = buildConjunctionClause(list,new ClauseDescription[]{
+      new UnitaryClause(jobIDField,jobID),
+      new UnitaryClause(statusField,statusToString(STATUS_COMPLETE))});
+    performUpdate(map,"WHERE "+query,list,null);
+    noteModifications(0,1,0);
+    // Do an analyze, otherwise our plans are going to be crap right off the bat
+    unconditionallyAnalyzeTables();
+  }
+  
   /** Prepare for an "incremental" job.  This is called ONLY when the job is inactive;
   * that is, there should be no ACTIVE or ACTIVEPURGATORY entries at all.
   *
@@ -603,7 +636,7 @@ public class JobQueue extends org.apache
   public void prepareIncrementalScan(Long jobID)
     throws ManifoldCFException
   {
-    // Delete PENDING and ACTIVE entries
+    // Map COMPLETE to PENDINGPURGATORY.
     HashMap map = new HashMap();
     map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
     // Do not reset priorities here!  They should all be blank at this point.
@@ -614,7 +647,9 @@ public class JobQueue extends org.apache
     ArrayList list = new ArrayList();
     String query = buildConjunctionClause(list,new ClauseDescription[]{
       new UnitaryClause(jobIDField,jobID),
-      new UnitaryClause(statusField,statusToString(STATUS_COMPLETE))});
+      new MultiClause(statusField,new Object[]{
+        statusToString(STATUS_COMPLETE),
+        statusToString(STATUS_UNCHANGED)})});
     performUpdate(map,"WHERE "+query,list,null);
     noteModifications(0,1,0);
     // Do an analyze, otherwise our plans are going to be crap right off the bat
@@ -1001,6 +1036,7 @@ public class JobQueue extends org.apache
       break;
 
     case STATUS_COMPLETE:
+    case STATUS_UNCHANGED:
     case STATUS_PURGATORY:
       // Set the status and time both
       map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
@@ -1278,6 +1314,7 @@ public class JobQueue extends org.apache
     switch (currentStatus)
     {
     case STATUS_PURGATORY:
+    case STATUS_UNCHANGED:
       // Set the status and time both
       map.put(statusField,statusToString(STATUS_PENDINGPURGATORY));
       map.put(checkTimeField,new Long(desiredExecuteTime));
@@ -1506,6 +1543,8 @@ public class JobQueue extends org.apache
       return "A";
     case STATUS_COMPLETE:
       return "C";
+    case STATUS_UNCHANGED:
+      return "U";
     case STATUS_PENDINGPURGATORY:
       return "G";
     case STATUS_ACTIVEPURGATORY:

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java Sat Mar 16 21:03:08 2013
@@ -92,17 +92,23 @@ public class Jobs extends org.apache.man
   public static final int STATUS_RESUMINGSEEDING = 17;               // In the process of resuming a paused or waited job, seeding process active too; will enter STATUS_ACTIVESEEDING when done.
   public static final int STATUS_ABORTING = 18;                          // Aborting (not yet aborted because documents still being processed)
   public static final int STATUS_STARTINGUP = 19;                        // Loading the queue (will go into ACTIVE if successful, or INACTIVE if not)
-  public static final int STATUS_ABORTINGSTARTINGUP = 20;        // Will abort once the queue loading is complete
-  public static final int STATUS_READYFORSTARTUP = 21;             // Job is marked for startup; startup thread has not taken it yet.
-  public static final int STATUS_READYFORDELETE = 22;             // Job is marked for delete; delete thread has not taken it yet.
-  public static final int STATUS_ABORTINGSEEDING = 23;            // Same as aborting, but seeding process is currently active also.
-  public static final int STATUS_ABORTINGFORRESTART = 24;       // Same as aborting, except after abort is complete startup will happen.
-  public static final int STATUS_ABORTINGFORRESTARTSEEDING = 25;  // Seeding version of aborting for restart
-  public static final int STATUS_ABORTINGSTARTINGUPFORRESTART = 26; // Starting up version of aborting for restart
-  public static final int STATUS_READYFORNOTIFY = 27;                   // Job is ready to be notified of completion
-  public static final int STATUS_NOTIFYINGOFCOMPLETION = 28;    // Notifying connector of terminating job (either aborted, or finished)
-  public static final int STATUS_DELETING = 29;                         // The job is deleting.
-  public static final int STATUS_DELETESTARTINGUP = 30;         // The delete is starting up.
+  public static final int STATUS_STARTINGUPMINIMAL = 20;           // Loading the queue for minimal job run (will go into ACTIVE if successful, or INACTIVE if not)
+  public static final int STATUS_ABORTINGSTARTINGUP = 21;        // Will abort once the queue loading is complete
+  public static final int STATUS_ABORTINGSTARTINGUPMINIMAL = 22;  // Will abort once the queue loading is complete
+  public static final int STATUS_READYFORSTARTUP = 23;             // Job is marked for minimal startup; startup thread has not taken it yet.
+  public static final int STATUS_READYFORSTARTUPMINIMAL = 24;   // Job is marked for startup; startup thread has not taken it yet.
+  public static final int STATUS_READYFORDELETE = 25;             // Job is marked for delete; delete thread has not taken it yet.
+  public static final int STATUS_ABORTINGSEEDING = 26;            // Same as aborting, but seeding process is currently active also.
+  public static final int STATUS_ABORTINGFORRESTART = 27;       // Same as aborting, except after abort is complete startup will happen.
+  public static final int STATUS_ABORTINGFORRESTARTMINIMAL = 28;  // Same as aborting, except after abort is complete startup will happen.
+  public static final int STATUS_ABORTINGFORRESTARTSEEDING = 29;  // Seeding version of aborting for restart
+  public static final int STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL = 30;  // Seeding version of aborting for restart
+  public static final int STATUS_ABORTINGSTARTINGUPFORRESTART = 31; // Starting up version of aborting for restart
+  public static final int STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL = 32; // Starting up version of aborting for restart
+  public static final int STATUS_READYFORNOTIFY = 33;                   // Job is ready to be notified of completion
+  public static final int STATUS_NOTIFYINGOFCOMPLETION = 34;    // Notifying connector of terminating job (either aborted, or finished)
+  public static final int STATUS_DELETING = 35;                         // The job is deleting.
+  public static final int STATUS_DELETESTARTINGUP = 36;         // The delete is starting up.
   
   // These statuses have to do with whether a job has an installed underlying connector or not.
   // There are two reasons to have a special state here: (1) if the behavior of the crawler differs, or (2) if the
@@ -113,13 +119,13 @@ public class Jobs extends org.apache.man
   // But, since there is no indication in the jobs table of an uninstalled connector for such jobs, the code which starts
   // jobs up (or otherwise would enter any state that has a corresponding special state) must check to see if the underlying
   // connector exists before deciding what state to put the job into.
-  public static final int STATUS_ACTIVE_UNINSTALLED = 31;               // Active, but repository connector not installed
-  public static final int STATUS_ACTIVESEEDING_UNINSTALLED = 32;   // Active and seeding, but repository connector not installed
-  public static final int STATUS_ACTIVE_NOOUTPUT = 33;                  // Active, but output connector not installed
-  public static final int STATUS_ACTIVESEEDING_NOOUTPUT = 34;       // Active and seeding, but output connector not installed
-  public static final int STATUS_ACTIVE_NEITHER = 35;                     // Active, but neither repository connector nor output connector installed
-  public static final int STATUS_ACTIVESEEDING_NEITHER = 36;          // Active and seeding, but neither repository connector nor output connector installed
-  public static final int STATUS_DELETING_NOOUTPUT = 37;                // Job is being deleted but there's no output connector installed
+  public static final int STATUS_ACTIVE_UNINSTALLED = 37;               // Active, but repository connector not installed
+  public static final int STATUS_ACTIVESEEDING_UNINSTALLED = 38;   // Active and seeding, but repository connector not installed
+  public static final int STATUS_ACTIVE_NOOUTPUT = 39;                  // Active, but output connector not installed
+  public static final int STATUS_ACTIVESEEDING_NOOUTPUT = 40;       // Active and seeding, but output connector not installed
+  public static final int STATUS_ACTIVE_NEITHER = 41;                     // Active, but neither repository connector nor output connector installed
+  public static final int STATUS_ACTIVESEEDING_NEITHER = 42;          // Active and seeding, but neither repository connector nor output connector installed
+  public static final int STATUS_DELETING_NOOUTPUT = 43;                // Job is being deleted but there's no output connector installed
 
   // Type field values
   public static final int TYPE_CONTINUOUS = IJobDescription.TYPE_CONTINUOUS;
@@ -194,13 +200,18 @@ public class Jobs extends org.apache.man
     statusMap.put("Z",new Integer(STATUS_PAUSEDWAIT));
     statusMap.put("X",new Integer(STATUS_ABORTING));
     statusMap.put("B",new Integer(STATUS_STARTINGUP));
+    statusMap.put("b",new Integer(STATUS_STARTINGUPMINIMAL));
     statusMap.put("Q",new Integer(STATUS_ABORTINGSTARTINGUP));
+    statusMap.put("q",new Integer(STATUS_ABORTINGSTARTINGUPMINIMAL));
     statusMap.put("C",new Integer(STATUS_READYFORSTARTUP));
+    statusMap.put("c",new Integer(STATUS_READYFORSTARTUPMINIMAL));
     statusMap.put("E",new Integer(STATUS_READYFORDELETE));
     statusMap.put("V",new Integer(STATUS_DELETESTARTINGUP));
     statusMap.put("e",new Integer(STATUS_DELETING));
     statusMap.put("Y",new Integer(STATUS_ABORTINGFORRESTART));
+    statusMap.put("M",new Integer(STATUS_ABORTINGFORRESTARTMINIMAL));
     statusMap.put("T",new Integer(STATUS_ABORTINGSTARTINGUPFORRESTART));
+    statusMap.put("t",new Integer(STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL));
 
     statusMap.put("a",new Integer(STATUS_ACTIVESEEDING));
     statusMap.put("x",new Integer(STATUS_ABORTINGSEEDING));
@@ -208,6 +219,7 @@ public class Jobs extends org.apache.man
     statusMap.put("w",new Integer(STATUS_ACTIVEWAITSEEDING));
     statusMap.put("z",new Integer(STATUS_PAUSEDWAITSEEDING));
     statusMap.put("y",new Integer(STATUS_ABORTINGFORRESTARTSEEDING));
+    statusMap.put("m",new Integer(STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL));
 
     statusMap.put("H",new Integer(STATUS_ACTIVEWAITING));
     statusMap.put("h",new Integer(STATUS_ACTIVEWAITINGSEEDING));
@@ -842,6 +854,15 @@ public class Jobs extends org.apache.man
       map.put(statusField,statusToString(STATUS_READYFORSTARTUP));
       performUpdate(map,"WHERE "+query,list,invKey);
 
+      // Starting up or aborting starting up goes back to just being ready
+      list.clear();
+      query = buildConjunctionClause(list,new ClauseDescription[]{
+        new MultiClause(statusField,new Object[]{
+          statusToString(STATUS_STARTINGUPMINIMAL),
+          statusToString(STATUS_ABORTINGSTARTINGUPMINIMAL)})});
+      map.put(statusField,statusToString(STATUS_READYFORSTARTUPMINIMAL));
+      performUpdate(map,"WHERE "+query,list,invKey);
+
       // Aborting starting up for restart state goes to ABORTINGFORRESTART
       list.clear();
       query = buildConjunctionClause(list,new ClauseDescription[]{
@@ -849,6 +870,13 @@ public class Jobs extends org.apache.man
       map.put(statusField,statusToString(STATUS_ABORTINGFORRESTART));
       performUpdate(map,"WHERE "+query,list,invKey);
 
+      // Aborting starting up for restart state goes to ABORTINGFORRESTART
+      list.clear();
+      query = buildConjunctionClause(list,new ClauseDescription[]{
+        new UnitaryClause(statusField,statusToString(STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL))});
+      map.put(statusField,statusToString(STATUS_ABORTINGFORRESTARTMINIMAL));
+      performUpdate(map,"WHERE "+query,list,invKey);
+
       // All seeding values return to pre-seeding values
       list.clear();
       query = buildConjunctionClause(list,new ClauseDescription[]{
@@ -887,6 +915,11 @@ public class Jobs extends org.apache.man
       performUpdate(map,"WHERE "+query,list,invKey);
       list.clear();
       query = buildConjunctionClause(list,new ClauseDescription[]{
+        new UnitaryClause(statusField,statusToString(STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL))});
+      map.put(statusField,statusToString(STATUS_ABORTINGFORRESTARTMINIMAL));
+      performUpdate(map,"WHERE "+query,list,invKey);
+      list.clear();
+      query = buildConjunctionClause(list,new ClauseDescription[]{
         new UnitaryClause(statusField,statusToString(STATUS_PAUSEDSEEDING))});
       map.put(statusField,statusToString(STATUS_PAUSED));
       performUpdate(map,"WHERE "+query,list,invKey);
@@ -1163,7 +1196,7 @@ public class Jobs extends org.apache.man
     int status = stringToStatus(statusValue);
     // Any active state in the lifecycle will do: seeding, active, active_seeding
     return (status == STATUS_ACTIVE || status == STATUS_ACTIVESEEDING ||
-      status == STATUS_STARTINGUP);
+      status == STATUS_STARTINGUP || status == STATUS_STARTINGUPMINIMAL);
   }
 
   /** Reset delete startup worker thread status.
@@ -1209,6 +1242,7 @@ public class Jobs extends org.apache.man
     HashMap map = new HashMap();
     String query;
 
+    list.clear();
     query = buildConjunctionClause(list,new ClauseDescription[]{
       new UnitaryClause(statusField,statusToString(STATUS_STARTINGUP))});
     map.put(statusField,statusToString(STATUS_READYFORSTARTUP));
@@ -1216,7 +1250,15 @@ public class Jobs extends org.apache.man
 
     list.clear();
     query = buildConjunctionClause(list,new ClauseDescription[]{
-      new UnitaryClause(statusField,statusToString(STATUS_ABORTINGSTARTINGUP))});
+      new UnitaryClause(statusField,statusToString(STATUS_STARTINGUPMINIMAL))});
+    map.put(statusField,statusToString(STATUS_READYFORSTARTUPMINIMAL));
+    performUpdate(map,"WHERE "+query,list,new StringSet(getJobStatusKey()));
+
+    list.clear();
+    query = buildConjunctionClause(list,new ClauseDescription[]{
+      new MultiClause(statusField,new Object[]{
+        statusToString(STATUS_ABORTINGSTARTINGUP),
+        statusToString(STATUS_ABORTINGSTARTINGUPMINIMAL)})});
     map.put(statusField,statusToString(STATUS_ABORTING));
     performUpdate(map,"WHERE "+query,list,new StringSet(getJobStatusKey()));
 
@@ -1226,6 +1268,12 @@ public class Jobs extends org.apache.man
     map.put(statusField,statusToString(STATUS_ABORTINGFORRESTART));
     performUpdate(map,"WHERE "+query,list,new StringSet(getJobStatusKey()));
 
+    list.clear();
+    query = buildConjunctionClause(list,new ClauseDescription[]{
+      new UnitaryClause(statusField,statusToString(STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL))});
+    map.put(statusField,statusToString(STATUS_ABORTINGFORRESTARTMINIMAL));
+    performUpdate(map,"WHERE "+query,list,new StringSet(getJobStatusKey()));
+
   }
 
   /** Reset as part of restoring seeding worker threads.
@@ -1278,6 +1326,11 @@ public class Jobs extends org.apache.man
       performUpdate(map,"WHERE "+query,list,invKey);
       list.clear();
       query = buildConjunctionClause(list,new ClauseDescription[]{
+        new UnitaryClause(statusField,statusToString(STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL))});
+      map.put(statusField,statusToString(STATUS_ABORTINGFORRESTARTMINIMAL));
+      performUpdate(map,"WHERE "+query,list,invKey);
+      list.clear();
+      query = buildConjunctionClause(list,new ClauseDescription[]{
         new UnitaryClause(statusField,statusToString(STATUS_PAUSEDSEEDING))});
       map.put(statusField,statusToString(STATUS_PAUSED));
       performUpdate(map,"WHERE "+query,list,invKey);
@@ -1329,15 +1382,16 @@ public class Jobs extends org.apache.man
   * when the job enters the "active" state.)
   *@param jobID is the job identifier.
   *@param windowEnd is the window end time, if any
+  *@param requestMinimum is true if a minimal job run is requested
   */
-  public void startJob(Long jobID, Long windowEnd)
+  public void startJob(Long jobID, Long windowEnd, boolean requestMinimum)
     throws ManifoldCFException
   {
     ArrayList list = new ArrayList();
     String query = buildConjunctionClause(list,new ClauseDescription[]{
       new UnitaryClause(idField,jobID)});
     HashMap map = new HashMap();
-    map.put(statusField,statusToString(STATUS_READYFORSTARTUP));
+    map.put(statusField,statusToString(requestMinimum?STATUS_READYFORSTARTUPMINIMAL:STATUS_READYFORSTARTUP));
     map.put(endTimeField,null);
     // Make sure error is removed (from last time)
     map.put(errorField,null);
@@ -1488,6 +1542,7 @@ public class Jobs extends org.apache.man
       switch (status)
       {
       case STATUS_STARTINGUP:
+      case STATUS_STARTINGUPMINIMAL:
         if (connectionMgr.checkConnectorExists((String)row.getValue(connectionNameField)))
         {
           if (outputMgr.checkConnectorExists((String)row.getValue(outputNameField)))
@@ -1504,11 +1559,15 @@ public class Jobs extends org.apache.man
         }
         break;
       case STATUS_ABORTINGSTARTINGUP:
+      case STATUS_ABORTINGSTARTINGUPMINIMAL:
         newStatus = STATUS_ABORTING;
         break;
       case STATUS_ABORTINGSTARTINGUPFORRESTART:
         newStatus = STATUS_ABORTINGFORRESTART;
         break;
+      case STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL:
+        newStatus = STATUS_ABORTINGFORRESTARTMINIMAL;
+        break;
       default:
         // Complain!
         throw new ManifoldCFException("Unexpected job status encountered: "+Integer.toString(status));
@@ -1600,6 +1659,9 @@ public class Jobs extends org.apache.man
       case STATUS_ABORTINGFORRESTARTSEEDING:
         newStatus = STATUS_ABORTINGFORRESTART;
         break;
+      case STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL:
+        newStatus = STATUS_ABORTINGFORRESTARTMINIMAL;
+        break;
       default:
         throw new ManifoldCFException("Unexpected job status encountered: "+Integer.toString(status));
       }
@@ -1677,7 +1739,8 @@ public class Jobs extends org.apache.man
       throw new ManifoldCFException("Job does not exist: "+jobID);
     IResultRow row = set.getRow(0);
     int status = stringToStatus(row.getValue(statusField).toString());
-    if (status == STATUS_ABORTING || status == STATUS_ABORTINGSEEDING || status == STATUS_ABORTINGSTARTINGUP)
+    if (status == STATUS_ABORTING || status == STATUS_ABORTINGSEEDING ||
+      status == STATUS_ABORTINGSTARTINGUP || status == STATUS_ABORTINGSTARTINGUPMINIMAL)
       return false;
     int newStatus;
     switch (status)
@@ -1686,7 +1749,12 @@ public class Jobs extends org.apache.man
     case STATUS_ABORTINGSTARTINGUPFORRESTART:
       newStatus = STATUS_ABORTINGSTARTINGUP;
       break;
+    case STATUS_STARTINGUPMINIMAL:
+    case STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL:
+      newStatus = STATUS_ABORTINGSTARTINGUPMINIMAL;
+      break;
     case STATUS_READYFORSTARTUP:
+    case STATUS_READYFORSTARTUPMINIMAL:
     case STATUS_ACTIVE:
     case STATUS_ACTIVE_UNINSTALLED:
     case STATUS_ACTIVE_NOOUTPUT:
@@ -1698,6 +1766,7 @@ public class Jobs extends org.apache.man
     case STATUS_PAUSED:
     case STATUS_PAUSEDWAIT:
     case STATUS_ABORTINGFORRESTART:
+    case STATUS_ABORTINGFORRESTARTMINIMAL:
       newStatus = STATUS_ABORTING;
       break;
     case STATUS_ACTIVESEEDING:
@@ -1711,6 +1780,7 @@ public class Jobs extends org.apache.man
     case STATUS_PAUSEDSEEDING:
     case STATUS_PAUSEDWAITSEEDING:
     case STATUS_ABORTINGFORRESTARTSEEDING:
+    case STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL:
       newStatus = STATUS_ABORTINGSEEDING;
       break;
     default:
@@ -1726,8 +1796,9 @@ public class Jobs extends org.apache.man
 
   /** Restart a job.  Finish off what's currently happening, and then start the job up again.
   *@param jobID is the job id.
+  *@param requestMinimum is true if the minimal job run is requested.
   */
-  public void abortRestartJob(Long jobID)
+  public void abortRestartJob(Long jobID, boolean requestMinimum)
     throws ManifoldCFException
   {
     // Get the current job status
@@ -1740,15 +1811,20 @@ public class Jobs extends org.apache.man
       throw new ManifoldCFException("Job does not exist: "+jobID);
     IResultRow row = set.getRow(0);
     int status = stringToStatus(row.getValue(statusField).toString());
-    if (status == STATUS_ABORTINGFORRESTART || status == STATUS_ABORTINGFORRESTARTSEEDING || status == STATUS_ABORTINGSTARTINGUPFORRESTART)
+    if (status == STATUS_ABORTINGFORRESTART || status == STATUS_ABORTINGFORRESTARTSEEDING ||
+      status == STATUS_ABORTINGSTARTINGUPFORRESTART ||
+      status == STATUS_ABORTINGFORRESTARTMINIMAL || status == STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL ||
+      status == STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL)
       return;
     int newStatus;
     switch (status)
     {
     case STATUS_STARTINGUP:
-      newStatus = STATUS_ABORTINGSTARTINGUPFORRESTART;
+    case STATUS_STARTINGUPMINIMAL:
+      newStatus = requestMinimum?STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL:STATUS_ABORTINGSTARTINGUPFORRESTART;
       break;
     case STATUS_READYFORSTARTUP:
+    case STATUS_READYFORSTARTUPMINIMAL:
     case STATUS_ACTIVE:
     case STATUS_ACTIVE_UNINSTALLED:
     case STATUS_ACTIVE_NOOUTPUT:
@@ -1759,7 +1835,7 @@ public class Jobs extends org.apache.man
     case STATUS_PAUSINGWAITING:
     case STATUS_PAUSED:
     case STATUS_PAUSEDWAIT:
-      newStatus = STATUS_ABORTINGFORRESTART;
+      newStatus = requestMinimum?STATUS_ABORTINGFORRESTARTMINIMAL:STATUS_ABORTINGFORRESTART;
       break;
     case STATUS_ACTIVESEEDING:
     case STATUS_ACTIVESEEDING_UNINSTALLED:
@@ -1771,7 +1847,7 @@ public class Jobs extends org.apache.man
     case STATUS_PAUSINGWAITINGSEEDING:
     case STATUS_PAUSEDSEEDING:
     case STATUS_PAUSEDWAITSEEDING:
-      newStatus = STATUS_ABORTINGFORRESTARTSEEDING;
+      newStatus = requestMinimum?STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL:STATUS_ABORTINGFORRESTARTSEEDING;
       break;
     default:
       throw new ManifoldCFException("Job "+jobID+" is not restartable");
@@ -2061,6 +2137,13 @@ public class Jobs extends org.apache.man
         map.put(errorField,null);
         map.put(windowEndField,null);
         break;
+      case STATUS_ABORTINGFORRESTARTMINIMAL:
+        map.put(statusField,statusToString(STATUS_READYFORSTARTUPMINIMAL));
+        map.put(endTimeField,null);
+        // Make sure error is removed (from last time)
+        map.put(errorField,null);
+        map.put(windowEndField,null);
+        break;
       case STATUS_PAUSING:
         map.put(statusField,statusToString(STATUS_PAUSED));
         break;
@@ -2295,14 +2378,24 @@ public class Jobs extends org.apache.man
       return "X";
     case STATUS_ABORTINGFORRESTART:
       return "Y";
+    case STATUS_ABORTINGFORRESTARTMINIMAL:
+      return "M";
     case STATUS_STARTINGUP:
       return "B";
+    case STATUS_STARTINGUPMINIMAL:
+      return "b";
     case STATUS_ABORTINGSTARTINGUP:
       return "Q";
+    case STATUS_ABORTINGSTARTINGUPMINIMAL:
+      return "q";
     case STATUS_ABORTINGSTARTINGUPFORRESTART:
       return "T";
+    case STATUS_ABORTINGSTARTINGUPFORRESTARTMINIMAL:
+      return "t";
     case STATUS_READYFORSTARTUP:
       return "C";
+    case STATUS_READYFORSTARTUPMINIMAL:
+      return "c";
     case STATUS_READYFORDELETE:
       return "E";
     case STATUS_DELETESTARTINGUP:
@@ -2321,6 +2414,8 @@ public class Jobs extends org.apache.man
       return "z";
     case STATUS_ABORTINGFORRESTARTSEEDING:
       return "y";
+    case STATUS_ABORTINGFORRESTARTSEEDINGMINIMAL:
+      return "m";
     case STATUS_ACTIVE_UNINSTALLED:
       return "R";
     case STATUS_ACTIVESEEDING_UNINSTALLED:

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/ScheduleManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/ScheduleManager.java?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/ScheduleManager.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/ScheduleManager.java Sat Mar 16 21:03:08 2013
@@ -41,6 +41,7 @@ import java.util.*;
 * <tr><td>minutesofhour</td><td>VARCHAR(255)</td><td></td></tr>
 * <tr><td>timezone</td><td>VARCHAR(32)</td><td></td></tr>
 * <tr><td>windowlength</td><td>BIGINT</td><td></td></tr>
+* <tr><td>requestminimum</td><td>CHAR(1)</td><td></td></tr>
 * </table>
 * <br><br>
 * 
@@ -60,7 +61,7 @@ public class ScheduleManager extends org
   public final static String minutesOfHourField = "minutesofhour";
   public final static String timezoneField = "timezone";
   public final static String windowDurationField = "windowlength";
-
+  public final static String requestMinimumField = "requestminimum";
 
   /** Constructor.
   *@param threadContext is the thread context.
@@ -96,6 +97,7 @@ public class ScheduleManager extends org
         map.put(minutesOfHourField,new ColumnDescription("VARCHAR(255)",false,true,null,null,false));
         map.put(timezoneField,new ColumnDescription("VARCHAR(32)",false,true,null,null,false));
         map.put(windowDurationField,new ColumnDescription("BIGINT",false,true,null,null,false));
+        map.put(requestMinimumField,new ColumnDescription("CHAR(1)",false,true,null,null,false));
         performCreate(map,null);
       }
       else
@@ -113,6 +115,14 @@ public class ScheduleManager extends org
           performAlter(null,null,list,null);
 
         }
+        
+        if (existing.get(requestMinimumField) == null)
+        {
+          HashMap map = new HashMap();
+          map.put(requestMinimumField,new ColumnDescription("CHAR(1)",false,true,null,null,false));
+          performAlter(map,null,null,null);
+        }
+        
       }
 
       // Index management
@@ -171,7 +181,8 @@ public class ScheduleManager extends org
         stringToEnumeratedValue((String)row.getValue(hourOfDayField)),
         stringToEnumeratedValue((String)row.getValue(minutesOfHourField)),
         (String)row.getValue(timezoneField),
-        (Long)row.getValue(windowDurationField));
+        (Long)row.getValue(windowDurationField),
+        stringToRequestMinimumValue((String)row.getValue(requestMinimumField)));
       ((JobDescription)returnValues.get(ownerID)).addScheduleRecord(sr);
       i++;
     }
@@ -209,7 +220,8 @@ public class ScheduleManager extends org
         stringToEnumeratedValue((String)row.getValue(hourOfDayField)),
         stringToEnumeratedValue((String)row.getValue(minutesOfHourField)),
         (String)row.getValue(timezoneField),
-        (Long)row.getValue(windowDurationField));
+        (Long)row.getValue(windowDurationField),
+        stringToRequestMinimumValue((String)row.getValue(requestMinimumField)));
       ArrayList theList = (ArrayList)returnValues.get(ownerID);
       if (theList == null)
       {
@@ -245,6 +257,7 @@ public class ScheduleManager extends org
         map.put(minutesOfHourField,enumeratedValueToString(record.getMinutesOfHour()));
         map.put(timezoneField,record.getTimezone());
         map.put(windowDurationField,record.getDuration());
+        map.put(requestMinimumField,requestMinimumValueToString(record.getRequestMinimum()));
         map.put(ownerIDField,ownerID);
         map.put(ordinalField,new Long((long)i));
         performInsert(map,null);
@@ -340,5 +353,22 @@ public class ScheduleManager extends org
     return rval.toString();
   }
 
-
+  public static String requestMinimumValueToString(boolean requestMinimum)
+  {
+    return requestMinimum?"T":"F";
+  }
+  
+  public static boolean stringToRequestMinimumValue(String requestMinimum)
+    throws ManifoldCFException
+  {
+    if (requestMinimum == null)
+      return false;
+    else if (requestMinimum.equals("T"))
+      return true;
+    else if (requestMinimum.equals("F"))
+      return false;
+    else
+      throw new ManifoldCFException("Bad requestminimum value: '"+requestMinimum+"'");
+  }
+    
 }

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobNotificationThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobNotificationThread.java?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobNotificationThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/JobNotificationThread.java Sat Mar 16 21:03:08 2013
@@ -66,7 +66,7 @@ public class JobNotificationThread exten
           // Before we begin, conditionally reset
           resetManager.waitForReset(threadContext);
 
-          JobStartRecord[] jobsNeedingNotification = jobManager.getJobsReadyForInactivity();
+          JobNotifyRecord[] jobsNeedingNotification = jobManager.getJobsReadyForInactivity();
           try
           {
             HashMap connectionNames = new HashMap();
@@ -74,7 +74,7 @@ public class JobNotificationThread exten
             int k = 0;
             while (k < jobsNeedingNotification.length)
             {
-              JobStartRecord jsr = jobsNeedingNotification[k++];
+              JobNotifyRecord jsr = jobsNeedingNotification[k++];
               Long jobID = jsr.getJobID();
               IJobDescription job = jobManager.load(jobID,true);
               if (job != null)
@@ -144,7 +144,7 @@ public class JobNotificationThread exten
             k = 0;
             while (k < jobsNeedingNotification.length)
             {
-              JobStartRecord jsr = jobsNeedingNotification[k++];
+              JobNotifyRecord jsr = jobsNeedingNotification[k++];
               Long jobID = jsr.getJobID();
               IJobDescription job = jobManager.load(jobID,true);
               if (job != null)
@@ -170,7 +170,7 @@ public class JobNotificationThread exten
             int i = 0;
             while (i < jobsNeedingNotification.length)
             {
-              JobStartRecord jsr = jobsNeedingNotification[i++];
+              JobNotifyRecord jsr = jobsNeedingNotification[i++];
               if (!jsr.wasStarted())
               {
                 // Clean up from failed start.

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java Sat Mar 16 21:03:08 2013
@@ -3118,13 +3118,13 @@ public class ManifoldCF extends org.apac
   
   /** Start a job.
   */
-  protected static int apiWriteStartJob(IThreadContext tc, Configuration output, Long jobID)
+  protected static int apiWriteStartJob(IThreadContext tc, Configuration output, Long jobID, boolean requestMinimum)
     throws ManifoldCFException
   {
     try
     {
       IJobManager jobManager = JobManagerFactory.make(tc);
-      jobManager.manualStart(jobID);
+      jobManager.manualStart(jobID,requestMinimum);
       return WRITERESULT_CREATED;
     }
     catch (ManifoldCFException e)
@@ -3154,13 +3154,13 @@ public class ManifoldCF extends org.apac
   
   /** Restart a job.
   */
-  protected static int apiWriteRestartJob(IThreadContext tc, Configuration output, Long jobID)
+  protected static int apiWriteRestartJob(IThreadContext tc, Configuration output, Long jobID, boolean requestMinimum)
     throws ManifoldCFException
   {
     try
     {
       IJobManager jobManager = JobManagerFactory.make(tc);
-      jobManager.manualAbortRestart(jobID);
+      jobManager.manualAbortRestart(jobID,requestMinimum);
       return WRITERESULT_CREATED;
     }
     catch (ManifoldCFException e)
@@ -3379,7 +3379,12 @@ public class ManifoldCF extends org.apac
     if (path.startsWith("start/"))
     {
       Long jobID = new Long(path.substring("start/".length()));
-      return apiWriteStartJob(tc,output,jobID);
+      return apiWriteStartJob(tc,output,jobID,false);
+    }
+    else if (path.startsWith("startminimal/"))
+    {
+      Long jobID = new Long(path.substring("startminimal/".length()));
+      return apiWriteStartJob(tc,output,jobID,true);
     }
     else if (path.startsWith("abort/"))
     {
@@ -3389,7 +3394,12 @@ public class ManifoldCF extends org.apac
     else if (path.startsWith("restart/"))
     {
       Long jobID = new Long(path.substring("restart/".length()));
-      return apiWriteRestartJob(tc,output,jobID);
+      return apiWriteRestartJob(tc,output,jobID,false);
+    }
+    else if (path.startsWith("restartminimal/"))
+    {
+      Long jobID = new Long(path.substring("restartminimal/".length()));
+      return apiWriteRestartJob(tc,output,jobID,true);
     }
     else if (path.startsWith("pause/"))
     {
@@ -3580,6 +3590,7 @@ public class ManifoldCF extends org.apac
   protected static final String JOBNODE_SCHEDULE = "schedule";
   protected static final String JOBNODE_LINKTYPE = "link_type";
   protected static final String JOBNODE_COUNT = "count";
+  protected static final String JOBNODE_REQUESTMINIMUM = "requestminimum";
   protected static final String JOBNODE_TIMEZONE = "timezone";
   protected static final String JOBNODE_DURATION = "duration";
   protected static final String JOBNODE_DAYOFWEEK = "dayofweek";
@@ -3732,6 +3743,7 @@ public class ManifoldCF extends org.apac
         // Create a schedule record.
         String timezone = null;
         Long duration = null;
+        boolean requestMinimum = false;
         EnumeratedValues dayOfWeek = null;
         EnumeratedValues monthOfYear = null;
         EnumeratedValues dayOfMonth = null;
@@ -3745,6 +3757,10 @@ public class ManifoldCF extends org.apac
         {
           ConfigurationNode scheduleField = child.findChild(q++);
           String fieldType = scheduleField.getType();
+          if (fieldType.equals(JOBNODE_REQUESTMINIMUM))
+          {
+            requestMinimum = scheduleField.getValue().equals("true");
+          }
           if (fieldType.equals(JOBNODE_TIMEZONE))
           {
             timezone = scheduleField.getValue();
@@ -3780,7 +3796,7 @@ public class ManifoldCF extends org.apac
           else
             throw new ManifoldCFException("Unrecognized field in schedule record: '"+fieldType+"'");
         }
-        ScheduleRecord sr = new ScheduleRecord(dayOfWeek,monthOfYear,dayOfMonth,year,hourOfDay,minutesOfHour,timezone,duration);
+        ScheduleRecord sr = new ScheduleRecord(dayOfWeek,monthOfYear,dayOfMonth,year,hourOfDay,minutesOfHour,timezone,duration,requestMinimum);
         // Add the schedule record to the job.
         jobDescription.addScheduleRecord(sr);
       }
@@ -3931,6 +3947,11 @@ public class ManifoldCF extends org.apac
       child = new ConfigurationNode(JOBNODE_SCHEDULE);
       ConfigurationNode recordChild;
       
+      // requestminimum
+      recordChild = new ConfigurationNode(JOBNODE_REQUESTMINIMUM);
+      recordChild.setValue(sr.getRequestMinimum()?"true":"false");
+      child.addChild(child.getChildCount(),recordChild);
+      
       // timezone
       if (sr.getTimezone() != null)
       {

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/SeedingThread.java Sat Mar 16 21:03:08 2013
@@ -91,7 +91,7 @@ public class SeedingThread extends Threa
           Logging.threads.debug("Seeding thread woke up");
 
           // Grab active, adaptive jobs (and set their state to xxxSEEDING as a side effect)
-          JobStartRecord[] seedJobs = jobManager.getJobsReadyForSeeding(currentTime);
+          JobSeedingRecord[] seedJobs = jobManager.getJobsReadyForSeeding(currentTime);
 
           // Process these jobs, and do the seeding.  The seeding is based on what came back
           // in the job start record for sync time.  If there's an interruption, we just go on
@@ -113,7 +113,7 @@ public class SeedingThread extends Threa
             int i = 0;
             while (i < seedJobs.length)
             {
-              JobStartRecord jsr = seedJobs[i++];
+              JobSeedingRecord jsr = seedJobs[i++];
               Long jobID = jsr.getJobID();
               try
               {
@@ -203,7 +203,7 @@ public class SeedingThread extends Threa
             int i = 0;
             while (i < seedJobs.length)
             {
-              JobStartRecord jsr = seedJobs[i++];
+              JobSeedingRecord jsr = seedJobs[i++];
               if (!jsr.wasStarted())
               {
                 if (Logging.threads.isDebugEnabled())

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartDeleteThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartDeleteThread.java?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartDeleteThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartDeleteThread.java Sat Mar 16 21:03:08 2013
@@ -80,7 +80,7 @@ public class StartDeleteThread extends T
 
           // See if there are any starting jobs.
           // Note: Since this following call changes the job state, we must be careful to reset it on any kind of failure.
-          JobStartRecord[] deleteJobs = jobManager.getJobsReadyForDelete();
+          JobDeleteRecord[] deleteJobs = jobManager.getJobsReadyForDelete();
           try
           {
 
@@ -100,7 +100,7 @@ public class StartDeleteThread extends T
             int i = 0;
             while (i < deleteJobs.length)
             {
-              JobStartRecord jsr = deleteJobs[i++];
+              JobDeleteRecord jsr = deleteJobs[i++];
               Long jobID = jsr.getJobID();
 	      try
 	      {
@@ -128,7 +128,7 @@ public class StartDeleteThread extends T
             int i = 0;
             while (i < deleteJobs.length)
             {
-              JobStartRecord jsr = deleteJobs[i++];
+              JobDeleteRecord jsr = deleteJobs[i++];
               if (!jsr.wasStarted())
               {
                 // Clean up from failed start.

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java (original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/StartupThread.java Sat Mar 16 21:03:08 2013
@@ -135,38 +135,15 @@ public class StartupThread extends Threa
                   // Get the number of link types.
                   String[] legalLinkTypes = connector.getRelationshipTypes();
 
-                  // The old logic here looked at the model, and if it was incomplete, either
-                  // performed a complete crawl initialization, or an incremental crawl initialization.
-                  // The fact is that we can now determine automatically what kind of crawl should be
-                  // done, based on the model the connector states and on the starting time that we
-                  // would be feeding it.  (The starting time is reset to 0 when the document specification
-                  // is changed - that's a crucial consideration.)
-                  //
-                  // The new logic does this:
-                  //
-                  // (1) If the connector has MODEL_ADD_CHANGE_DELETE, then
-                  // we let the connector run the show; there's no purge phase, and therefore the
-                  // documents are left in a COMPLETED state if they don't show up in the list
-                  // of seeds that require the attention of the connector.
-                  //
-                  // (2) If the connector has MODEL_ALL, then it's a full crawl no matter what, so
-                  // we do a full scan initialization.
-                  //
-                  // (3) If the connector has some other model, we look at the start time.  A start
-                  // time of 0 implies a full scan, while any other start time implies an incremental
-                  // scan.
-
-                  if (model != connector.MODEL_ADD_CHANGE_DELETE)
-                  {
-                    if (Logging.threads.isDebugEnabled())
-                      Logging.threads.debug("Preparing job "+jobID.toString()+" for execution...");
-                    if (jobType != IJobDescription.TYPE_CONTINUOUS && model != connector.MODEL_PARTIAL && (model == connector.MODEL_ALL || lastJobTime == 0L))
-                      jobManager.prepareFullScan(jobID,legalLinkTypes,hopcountMethod);
-                    else
-                      jobManager.prepareIncrementalScan(jobID,legalLinkTypes,hopcountMethod);
-                    if (Logging.threads.isDebugEnabled())
-                      Logging.threads.debug("Prepared job "+jobID.toString()+" for execution.");
-                  }
+                  boolean requestMinimum = jsr.getRequestMinimum();
+                  
+                  if (Logging.threads.isDebugEnabled())
+                    Logging.threads.debug("Preparing job "+jobID.toString()+" for execution...");
+                  jobManager.prepareJobScan(jobID,legalLinkTypes,hopcountMethod,
+                    model,jobType == IJobDescription.TYPE_CONTINUOUS,lastJobTime == 0L,
+                    requestMinimum);
+                  if (Logging.threads.isDebugEnabled())
+                    Logging.threads.debug("Prepared job "+jobID.toString()+" for execution.");
 
                   try
                   {

Modified: manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties (original)
+++ manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_en_US.properties Sat Mar 16 21:03:08 2013
@@ -34,10 +34,6 @@ editjob.Scheduling=Scheduling
 editjob.HopFilters=Hop Filters
 editjob.am=am
 editjob.pm=pm
-editjob.st=st
-editjob.nd=nd
-editjob.rd=rd
-editjob.th=th
 
 index.WelcomeToApacheManifoldFC=Welcome to Apache ManifoldCF
 
@@ -341,6 +337,14 @@ editjob.NoForcedMetadataSpecified=No for
 editjob.Add=Add
 editjob.Addforcedmetadata=Add forced metadata
 editjob.ForcedMetadataNameMustNotBeNull=Forced metadata name must not be null
+editjob.st=st
+editjob.nd=nd
+editjob.rd=rd
+editjob.th=th
+editjob.dayofmonth=day of month
+editjob.JobInvocationColon=Job invocation:
+editjob.Minimal=Minimal
+editjob.Complete=Complete
 
 editjob.tab=tab
 
@@ -357,6 +361,39 @@ showjobstatus.Refresh=Refresh
 showjobstatus.PleaseTryAgainLater=This page is unavailable due to maintenance operations.  Please try again later.
 showjobstatus.StatusOfJobs=Status of Jobs
 
+showjobstatus.Notyetrun=Not yet run
+showjobstatus.Running=Running
+showjobstatus.Runningnoconnector=Running, no connector
+showjobstatus.Aborting=Aborting
+showjobstatus.Restarting=Restarting
+showjobstatus.Stopping=Stopping
+showjobstatus.Resuming=Resuming
+showjobstatus.Paused=Paused
+showjobstatus.Done=Done
+showjobstatus.Waiting=Waiting
+showjobstatus.Startingup=Starting up
+showjobstatus.Cleaningup=Cleaning up
+showjobstatus.Terminating=Terminating
+showjobstatus.Endnotification=End notification
+showjobstatus.ErrorColon=Error:
+showjobstatus.Unknown=Unknown
+showjobstatus.Notstarted=Not started
+showjobstatus.Aborted=Aborted
+showjobstatus.Neverrun=Never run
+showjobstatus.Start=Start
+showjobstatus.Startminimal=Start minimal
+showjobstatus.Restart=Restart
+showjobstatus.Restartminimal=Restart minimal
+showjobstatus.Pause=Pause
+showjobstatus.Abort=Abort
+showjobstatus.Resume=Resume
+showjobstatus.Startjob=Start job
+showjobstatus.minimally=minimally
+showjobstatus.Restartjob=Restart job
+showjobstatus.Pausejob=Pause job
+showjobstatus.Abortjob=Abort job
+showjobstatus.Resumejob=Resume job
+
 documentstatus.ApacheManifoldCFDocumentStatus=Apache ManifoldCF: Document Status
 documentstatus.DocumentStatus=Document Status
 documentstatus.Connection=Connection:
@@ -686,7 +723,7 @@ viewjob.at=at
 viewjob.am=am
 viewjob.pm=pm
 viewjob.plus=plus
-viewjob.inJanuary=in January
+viewjob.ineverymonthofyear=in every month of the year
 viewjob.in=in
 viewjob.January=January
 viewjob.February=February
@@ -713,6 +750,9 @@ viewjob.st=st
 viewjob.nd=nd
 viewjob.rd=rd
 viewjob.th=th
+viewjob.JobInvocationColon=Job invocation:
+viewjob.Minimal=Minimal
+viewjob.Complete=Complete
 
 viewauthority.ViewAuthorityConnectionStatus=View Authority Connection Status
 viewauthority.NameColon=Name:

Modified: manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties (original)
+++ manifoldcf/trunk/framework/ui-core/src/main/native2ascii/org/apache/manifoldcf/ui/i18n/common_ja_JP.properties Sat Mar 16 21:03:08 2013
@@ -34,10 +34,6 @@ editjob.Scheduling=スケジã�
 editjob.HopFilters=ホップフィルタ
 editjob.am=午前
 editjob.pm=午後
-editjob.st=日
-editjob.nd=日
-editjob.rd=日
-editjob.th=日
 
 index.WelcomeToApacheManifoldFC=Apache ManifoldCFへようこそ
 
@@ -341,6 +337,14 @@ editjob.NoForcedMetadataSpecified=No for
 editjob.Add=Add
 editjob.Addforcedmetadata=Add forced metadata
 editjob.ForcedMetadataNameMustNotBeNull=Forced metadata name must not be null
+editjob.st=日
+editjob.nd=日
+editjob.rd=日
+editjob.th=日
+editjob.dayofmonth=day of month
+editjob.JobInvocationColon=Job invocation:
+editjob.Minimal=Minimal
+editjob.Complete=Complete
 
 editjob.tab=tab
 
@@ -357,6 +361,39 @@ showjobstatus.Refresh=更新
 showjobstatus.PleaseTryAgainLater=保守処理中です。少々お待ちください。
 showjobstatus.StatusOfJobs=ジョブの状態
 
+showjobstatus.Notyetrun=Not yet run
+showjobstatus.Running=Running
+showjobstatus.Runningnoconnector=Running, no connector
+showjobstatus.Aborting=Aborting
+showjobstatus.Restarting=Restarting
+showjobstatus.Stopping=Stopping
+showjobstatus.Resuming=Resuming
+showjobstatus.Paused=Paused
+showjobstatus.Done=Done
+showjobstatus.Waiting=Waiting
+showjobstatus.Startingup=Starting up
+showjobstatus.Cleaningup=Cleaning up
+showjobstatus.Terminating=Terminating
+showjobstatus.Endnotification=End notification
+showjobstatus.ErrorColon=Error:
+showjobstatus.Unknown=Unknown
+showjobstatus.Notstarted=Not started
+showjobstatus.Aborted=Aborted
+showjobstatus.Neverrun=Never run
+showjobstatus.Start=Start
+showjobstatus.Startminimal=Start minimal
+showjobstatus.Restart=Restart
+showjobstatus.Restartminimal=Restart minimal
+showjobstatus.Pause=Pause
+showjobstatus.Abort=Abort
+showjobstatus.Resume=Resume
+showjobstatus.Startjob=Start job
+showjobstatus.minimally=minimally
+showjobstatus.Restartjob=Restart job
+showjobstatus.Pausejob=Pause job
+showjobstatus.Abortjob=Abort job
+showjobstatus.Resumejob=Resume job
+
 documentstatus.ApacheManifoldCFDocumentStatus=Apache ManifoldCF:コンテンツの状態
 documentstatus.DocumentStatus=コンテンツの状態
 documentstatus.Connection=コネクション:
@@ -673,34 +710,34 @@ viewjob.Scaneverydocumentonce=Scan every
 viewjob.Startatbeginningofschedulewindow=Start at beginning of schedule window
 viewjob.Startinsideschedulewindow=Start inside schedule window
 viewjob.Dontautomaticallystart=Don't automatically start
-viewjob.Anydayoftheweek=Any day of week
-viewjob.Sundays=Sundays
-viewjob.Mondays=Mondays
-viewjob.Tuesdays=Tuesdays
-viewjob.Wednesdays=Wednesdays
-viewjob.Thursdays=Thursdays
-viewjob.Fridays=Fridays
-viewjob.Saturdays=Saturdays
+viewjob.Anydayoftheweek=すべての曜日
+viewjob.Sundays=日
+viewjob.Mondays=月
+viewjob.Tuesdays=火
+viewjob.Wednesdays=æ°´
+viewjob.Thursdays=木
+viewjob.Fridays=金
+viewjob.Saturdays=土
 viewjob.oneveryhour=on every hour
 viewjob.atmidnight=at midnight
 viewjob.at=at
-viewjob.am=am
-viewjob.pm=pm
+viewjob.am=午前
+viewjob.pm=午後
 viewjob.plus=plus
-viewjob.inJanuary=in January
-viewjob.in=in
-viewjob.January=January
-viewjob.February=February
-viewjob.March=March
-viewjob.April=April
-viewjob.May=May
-viewjob.June=June
-viewjob.July=July
-viewjob.August=August
-viewjob.September=September
-viewjob.October=October
-viewjob.November=November
-viewjob.December=December
+viewjob.ineverymonthofyear=すべての月
+viewjob.in=
+viewjob.January=1月
+viewjob.February=2月
+viewjob.March=3月
+viewjob.April=4月
+viewjob.May=5月
+viewjob.June=6月
+viewjob.July=7月
+viewjob.August=8月
+viewjob.September=9月
+viewjob.October=10月
+viewjob.November=11月
+viewjob.December=12月
 viewjob.onthe1stofthemonth=on the 1st of the month
 viewjob.onthe=on the
 viewjob.ofthemonth=of the month
@@ -710,10 +747,13 @@ viewjob.Unlimited=Unlimited
 viewjob.Deleteunreachabledocuments=Delete unreachable documents
 viewjob.Nodeletesfornow=No deletes, for now
 viewjob.Nodeletesforever=No deletes, forever
-viewjob.st=st
-viewjob.nd=nd
-viewjob.rd=rd
-viewjob.th=th
+viewjob.st=日
+viewjob.nd=日
+viewjob.rd=日
+viewjob.th=日
+viewjob.JobInvocationColon=Job invocation:
+viewjob.Minimal=Minimal
+viewjob.Complete=Complete
 
 viewauthority.ViewAuthorityConnectionStatus=権限コネクション状態の表示
 viewauthority.NameColon=名前:

Modified: manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/end-user-documentation.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/end-user-documentation.xml?rev=1457305&r1=1457304&r2=1457305&view=diff
==============================================================================
--- manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/end-user-documentation.xml (original)
+++ manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/end-user-documentation.xml Sat Mar 16 21:03:08 2013
@@ -284,15 +284,20 @@
                        status column.  Allowed actions you may see at one point or another include:</p>
                 <ul>
                     <li>Start (start the job)</li>
+                    <li>Start minimal (start the job, but do only the minimal work possible)</li>
                     <li>Abort(abort the job)</li>
                     <li>Pause (pause the job)</li>
                     <li>Resume (resume the job)</li>
                     <li>Restart (equivalent to aborting the job, and starting it all over again)</li>
+                    <li>Restart minimal (equivalent to aborting the job, and starting it all over again, doing only the minimal work possible)</li>
                 </ul>
                 <br/>
                 <p>The columns "Documents", "Active", and "Processed" have very specific means as far as documents in the job's queue are concerned.  The "Documents" column counts all the documents
                        that belong to the job.  The "Active" column counts all of the documents for that job that are queued up for processing.  The "Processed" column counts all documents that are on the
                        queue for the job that have been processed at least once in the past.</p>
+                <p>Using the "minimal" variant of the listed actions will perform the minimum possible amount of work, given the model that the connection type for the job uses.  In some
+                       cases, this will mean that additions and modifications are indexed, but deletions are not detected.  A complete job run is usually necessary to fully synchronize the target
+                       index with the repository contents.</p>
             </section>
             <section id="statusreports">
                 <title>Status Reports</title>



Mime
View raw message