manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1529992 - in /manifoldcf/trunk: ./ framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ site/src/documentation/content/xdocs/en_US/ site/src/documentation/content/xdocs/ja_JP/
Date Mon, 07 Oct 2013 16:21:13 GMT
Author: kwright
Date: Mon Oct  7 16:21:12 2013
New Revision: 1529992

URL: http://svn.apache.org/r1529992
Log:
Finish CONNECTORS-783.

Modified:
    manifoldcf/trunk/CHANGES.txt
    manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
    manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml
    manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/programmatic-operation.xml
    manifoldcf/trunk/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml

Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1529992&r1=1529991&r2=1529992&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Mon Oct  7 16:21:12 2013
@@ -3,6 +3,10 @@ $Id$
 
 ======================= 1.4-dev =====================
 
+CONNECTORS-783: Add support for limiting the amount of counting
+needed for the job status page.
+(Hiroshi Tatsumi, Karl Wright)
+
 CONNECTORS-782: Add unique-ID metadata in SharePoint connector.
 (Dmitry Goldenberg, Karl Wright)
 

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java?rev=1529992&r1=1529991&r2=1529992&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ManifoldCF.java
Mon Oct  7 16:21:12 2013
@@ -2001,13 +2001,45 @@ public class ManifoldCF extends org.apac
   }
 
   /** Get api job statuses */
-  protected static int apiReadJobStatuses(IThreadContext tc, Configuration output)
+  protected static int apiReadJobStatuses(IThreadContext tc, Configuration output, Map<String,List<String>>
queryParameters)
     throws ManifoldCFException
   {
+    int maxCount;
+    List<String> maxCountList = queryParameters.get("maxcount");
+    if (maxCountList == null || maxCountList.size() == 0)
+      maxCount = Integer.MAX_VALUE;
+    else if (maxCountList.size() > 1)
+      throw new ManifoldCFException("Multiple values for maxcount parameter");
+    else
+      maxCount = new Integer(maxCountList.get(0)).intValue();
+      
     try
     {
       IJobManager jobManager = JobManagerFactory.make(tc);
-      JobStatus[] jobStatuses = jobManager.getAllStatus();
+      JobStatus[] jobStatuses = jobManager.getAllStatus(true,maxCount);
+      int i = 0;
+      while (i < jobStatuses.length)
+      {
+        ConfigurationNode jobStatusNode = new ConfigurationNode(API_JOBSTATUSNODE);
+        formatJobStatus(jobStatusNode,jobStatuses[i++]);
+        output.addChild(output.getChildCount(),jobStatusNode);
+      }
+    }
+    catch (ManifoldCFException e)
+    {
+      createErrorNode(output,e);
+    }
+    return READRESULT_FOUND;
+  }
+
+  /** Get api job statuses */
+  protected static int apiReadJobStatusesNoCounts(IThreadContext tc, Configuration output)
+    throws ManifoldCFException
+  {
+    try
+    {
+      IJobManager jobManager = JobManagerFactory.make(tc);
+      JobStatus[] jobStatuses = jobManager.getAllStatus(false);
       int i = 0;
       while (i < jobStatuses.length)
       {
@@ -2024,13 +2056,22 @@ public class ManifoldCF extends org.apac
   }
   
   /** Get api job status */
-  protected static int apiReadJobStatus(IThreadContext tc, Configuration output, Long jobID)
+  protected static int apiReadJobStatus(IThreadContext tc, Configuration output, Long jobID,
Map<String,List<String>> queryParameters)
     throws ManifoldCFException
   {
+    int maxCount;
+    List<String> maxCountList = queryParameters.get("maxcount");
+    if (maxCountList == null || maxCountList.size() == 0)
+      maxCount = Integer.MAX_VALUE;
+    else if (maxCountList.size() > 1)
+      throw new ManifoldCFException("Multiple values for maxcount parameter");
+    else
+      maxCount = new Integer(maxCountList.get(0)).intValue();
+
     try
     {
       IJobManager jobManager = JobManagerFactory.make(tc);
-      JobStatus status = jobManager.getStatus(jobID);
+      JobStatus status = jobManager.getStatus(jobID,true,maxCount);
       if (status != null)
       {
         ConfigurationNode jobStatusNode = new ConfigurationNode(API_JOBSTATUSNODE);
@@ -3147,12 +3188,16 @@ public class ManifoldCF extends org.apac
     }
     else if (path.equals("jobstatuses"))
     {
-      return apiReadJobStatuses(tc,output);
+      return apiReadJobStatuses(tc,output,queryParameters);
     }
     else if (path.startsWith("jobstatuses/"))
     {
       Long jobID = new Long(path.substring("jobstatuses/".length()));
-      return apiReadJobStatus(tc,output,jobID);
+      return apiReadJobStatus(tc,output,jobID,queryParameters);
+    }
+    else if (path.equals("jobstatusesnocounts"))
+    {
+      return apiReadJobStatusesNoCounts(tc,output);
     }
     else if (path.startsWith("jobstatusesnocounts/"))
     {
@@ -4348,6 +4393,9 @@ public class ManifoldCF extends org.apac
   protected static final String JOBSTATUSNODE_DOCUMENTSINQUEUE = "documents_in_queue";
   protected static final String JOBSTATUSNODE_DOCUMENTSOUTSTANDING = "documents_outstanding";
   protected static final String JOBSTATUSNODE_DOCUMENTSPROCESSED = "documents_processed";
+  protected static final String JOBSTATUSNODE_QUEUEEXACT = "queue_exact";
+  protected static final String JOBSTATUSNODE_OUTSTANDINGEXACT = "outstanding_exact";
+  protected static final String JOBSTATUSNODE_PROCESSEDEXACT = "processed_exact";
   
   /** Format a job status.
   */
@@ -4406,6 +4454,19 @@ public class ManifoldCF extends org.apac
     child.setValue(new Long(jobStatus.getDocumentsProcessed()).toString());
     jobStatusNode.addChild(jobStatusNode.getChildCount(),child);
 
+    // Exact flags
+    child = new ConfigurationNode(JOBSTATUSNODE_QUEUEEXACT);
+    child.setValue(new Boolean(jobStatus.getQueueCountExact()).toString());
+    jobStatusNode.addChild(jobStatusNode.getChildCount(),child);
+
+    child = new ConfigurationNode(JOBSTATUSNODE_OUTSTANDINGEXACT);
+    child.setValue(new Boolean(jobStatus.getOutstandingCountExact()).toString());
+    jobStatusNode.addChild(jobStatusNode.getChildCount(),child);
+
+    child = new ConfigurationNode(JOBSTATUSNODE_PROCESSEDEXACT);
+    child.setValue(new Boolean(jobStatus.getProcessedCountExact()).toString());
+    jobStatusNode.addChild(jobStatusNode.getChildCount(),child);
+
   }
 
   protected static String statusMap(int status)

Modified: manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml?rev=1529992&r1=1529991&r2=1529992&view=diff
==============================================================================
--- manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml
(original)
+++ manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/how-to-build-and-deploy.xml
Mon Oct  7 16:21:12 2013
@@ -803,6 +803,7 @@ cd example
             <tr><td>org.apache.manifoldcf.connectorsconfigurationfile</td><td>No</td><td>Location
of connectors.xml file, for QuickStart, so ManifoldCF can register connectors.</td></tr>
             <tr><td>org.apache.manifoldcf.dbsuperusername</td><td>No</td><td>Database
superuser name, for QuickStart, so ManifoldCF can create database instance.</td></tr>
             <tr><td>org.apache.manifoldcf.dbsuperuserpassword</td><td>No</td><td>Database
superuser password, for QuickStart, so ManifoldCF can create database instance.</td></tr>
+            <tr><td>org.apache.manifoldcf.ui.maxstatuscount</td><td>No</td><td>The
maximum number of documents ManifoldCF will try to count for the job status display.  Defaults
to 500000.</td></tr>
             <tr><td>org.apache.manifoldcf.postgresql.hostname</td><td>No</td><td>PostgreSQL
server host name, or localhost if not specified.</td></tr>
             <tr><td>org.apache.manifoldcf.postgresql.port</td><td>No</td><td>PostgreSQL
server port, or standard port if not specified.</td></tr>
             <tr><td>org.apache.manifoldcf.postgresql.ssl</td><td>No</td><td>Set
to "true" for ssl communication with PostgreSQL.</td></tr>

Modified: manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/programmatic-operation.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/programmatic-operation.xml?rev=1529992&r1=1529991&r2=1529992&view=diff
==============================================================================
--- manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/programmatic-operation.xml
(original)
+++ manifoldcf/trunk/site/src/documentation/content/xdocs/en_US/programmatic-operation.xml
Mon Oct  7 16:21:12 2013
@@ -111,8 +111,9 @@
             <tr><td>jobs/<em>&lt;job_id&gt;</em></td><td>GET</td><td>Get
a specific job definition</td><td>N/A</td><td>{"job":<em>&lt;job_object_&gt;</em>}
<strong>OR</strong> { } <strong>OR</strong> {"error":<em>&lt;error_text&gt;</em>}</td></tr>
             <tr><td>jobs/<em>&lt;job_id&gt;</em></td><td>PUT</td><td>Save
a job definition</td><td>{"job":<em>&lt;job_object&gt;</em>}</td><td>{
} <strong>OR</strong> {"error":<em>&lt;error_text&gt;</em>}</td></tr>
             <tr><td>jobs/<em>&lt;job_id&gt;</em></td><td>DELETE</td><td>Delete
a job definition</td><td>N/A</td><td>{ } <strong>OR</strong>
{"error":<em>&lt;error_text&gt;</em>}</td></tr>
-            <tr><td>jobstatuses</td><td>GET</td><td>List
all jobs and their status</td><td>N/A</td><td>{"jobstatus":[<em>&lt;list_of_job_status_objects&gt;</em>]}
<strong>OR</strong> {"error":<em>&lt;error_text&gt;</em>}</td></tr>
-            <tr><td>jobstatuses/<em>&lt;job_id&gt;</em></td><td>GET</td><td>Get
a specific job's status</td><td>N/A</td><td>{"jobstatus":<em>&lt;job_status_object&gt;</em>}
<strong>OR</strong> { } <strong>OR</strong> {"error":<em>&lt;error_text&gt;</em>}
</td></tr>
+            <tr><td>jobstatuses</td><td>GET</td><td>List
all jobs and their status</td><td>maxcount=&lt;maximum_documents_to_count&gt;</td><td>{"jobstatus":[<em>&lt;list_of_job_status_objects&gt;</em>]}
<strong>OR</strong> {"error":<em>&lt;error_text&gt;</em>}</td></tr>
+            <tr><td>jobstatuses/<em>&lt;job_id&gt;</em></td><td>GET</td><td>Get
a specific job's status</td><td>maxcount=&lt;maximum_documents_to_count&gt;</td><td>{"jobstatus":<em>&lt;job_status_object&gt;</em>}
<strong>OR</strong> { } <strong>OR</strong> {"error":<em>&lt;error_text&gt;</em>}
</td></tr>
+            <tr><td>jobstatusesnocounts<em>&lt;job_id&gt;</em></td><td>GET</td><td>List
all jobs and their status, returning '0' for all counts</td><td>N/A</td><td>{"jobstatus":[<em>&lt;list_of_job_status_objects&gt;</em>]}
<strong>OR</strong> { } <strong>OR</strong> {"error":<em>&lt;error_text&gt;</em>}
</td></tr>
             <tr><td>jobstatusesnocounts/<em>&lt;job_id&gt;</em></td><td>GET</td><td>Get
a specific job's status, returning '0' for all counts</td><td>N/A</td><td>{"jobstatus":<em>&lt;job_status_object&gt;</em>}
<strong>OR</strong> { } <strong>OR</strong> {"error":<em>&lt;error_text&gt;</em>}
</td></tr>
             <tr><td>start/<em>&lt;job_id&gt;</em></td><td>PUT</td><td>Start
a specified job manually</td><td>N/A</td><td>{ } <strong>OR</strong>
{"error":<em>&lt;error_text&gt;</em>}</td></tr>
             <tr><td>startminimal/<em>&lt;job_id&gt;</em></td><td>PUT</td><td>Start
a specified job manually, minimal run requested</td><td>N/A</td><td>{
} <strong>OR</strong> {"error":<em>&lt;error_text&gt;</em>}</td></tr>

Modified: manifoldcf/trunk/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml?rev=1529992&r1=1529991&r2=1529992&view=diff
==============================================================================
--- manifoldcf/trunk/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml
(original)
+++ manifoldcf/trunk/site/src/documentation/content/xdocs/ja_JP/how-to-build-and-deploy.xml
Mon Oct  7 16:21:12 2013
@@ -554,6 +554,14 @@ cd example
               correct arguments and settings.</p>
             <p></p>
             <table>
+              <tr><th>User Mapping Command Class</th><th>Arguments</th><th>Function</th></tr>
+              <tr><td>org.apache.manifoldcf.authorities.RegisterMapper</td><td><em>classname</em>
<em>description</em></td><td>Register a mapping connector class</td></tr>
+              <tr><td>org.apache.manifoldcf.authorities.UnRegisterMapper</td><td><em>classname</em></td><td>Un-register
a mapping connector class</td></tr>
+              <tr><td>org.apache.manifoldcf.authorities.UnRegisterAllMappers</td><td>None</td><td>Un-register
all mapping connector classes</td></tr>
+              <tr><td>org.apache.manifoldcf.authorities.SynchronizeMappers</td><td>None</td><td>Un-register
all registered mapping connector classes that can't be found</td></tr>
+            </table>
+            <p></p>
+            <table>
               <tr><th>Authority Command Class</th><th>Arguments</th><th>Function</th></tr>
               <tr><td>org.apache.manifoldcf.authorities.RegisterAuthority</td><td><em>classname</em>
<em>description</em></td><td>Register an authority connector class</td></tr>
               <tr><td>org.apache.manifoldcf.authorities.UnRegisterAuthority</td><td><em>classname</em></td><td>Un-register
an authority connector class</td></tr>
@@ -785,6 +793,8 @@ cd example
           <table>
             <caption>Property.xml properties</caption>
             <tr><th>Property</th><th>Required?</th><th>Function</th></tr>
+            <tr><td>org.apache.manifoldcf.login.name</td><td>No</td><td>Crawler
UI login user ID (defaults to "admin")</td></tr>
+            <tr><td>org.apache.manifoldcf.login.password</td><td>No</td><td>Crawler
UI login user password (defaults to "admin")</td></tr>
             <tr><td>org.apache.manifoldcf.crawleruiwarpath</td><td>Yes,
for Jetty</td><td>Location of Crawler UI war</td></tr>
             <tr><td>org.apache.manifoldcf.authorityservicewarpath</td><td>Yes,
for Jetty</td><td>Location of Authority Service war</td></tr>
             <tr><td>org.apache.manifoldcf.apiservicewarpath</td><td>Yes,
for Jetty</td><td>Location of API Service war</td></tr>
@@ -793,6 +803,7 @@ cd example
             <tr><td>org.apache.manifoldcf.connectorsconfigurationfile</td><td>No</td><td>Location
of connectors.xml file, for QuickStart, so ManifoldCF can register connectors.</td></tr>
             <tr><td>org.apache.manifoldcf.dbsuperusername</td><td>No</td><td>Database
superuser name, for QuickStart, so ManifoldCF can create database instance.</td></tr>
             <tr><td>org.apache.manifoldcf.dbsuperuserpassword</td><td>No</td><td>Database
superuser password, for QuickStart, so ManifoldCF can create database instance.</td></tr>
+            <tr><td>org.apache.manifoldcf.ui.maxstatuscount</td><td>No</td><td>The
maximum number of documents ManifoldCF will try to count for the job status display.  Defaults
to 500000.</td></tr>
             <tr><td>org.apache.manifoldcf.postgresql.hostname</td><td>No</td><td>PostgreSQL
server host name, or localhost if not specified.</td></tr>
             <tr><td>org.apache.manifoldcf.postgresql.port</td><td>No</td><td>PostgreSQL
server port, or standard port if not specified.</td></tr>
             <tr><td>org.apache.manifoldcf.postgresql.ssl</td><td>No</td><td>Set
to "true" for ssl communication with PostgreSQL.</td></tr>



Mime
View raw message