manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1529845 - in /manifoldcf/trunk/framework: crawler-ui/src/main/webapp/showjobstatus.jsp pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Date Mon, 07 Oct 2013 14:06:14 GMT
Author: kwright
Date: Mon Oct  7 14:06:14 2013
New Revision: 1529845

URL: http://svn.apache.org/r1529845
Log:
Hook up limits in the UI, through a parameter.

Modified:
    manifoldcf/trunk/framework/crawler-ui/src/main/webapp/showjobstatus.jsp
    manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java

Modified: manifoldcf/trunk/framework/crawler-ui/src/main/webapp/showjobstatus.jsp
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/crawler-ui/src/main/webapp/showjobstatus.jsp?rev=1529845&r1=1529844&r2=1529845&view=diff
==============================================================================
--- manifoldcf/trunk/framework/crawler-ui/src/main/webapp/showjobstatus.jsp (original)
+++ manifoldcf/trunk/framework/crawler-ui/src/main/webapp/showjobstatus.jsp Mon Oct  7 14:06:14
2013
@@ -110,9 +110,11 @@ if (maintenanceUnderway == false)
 <%
     try
     {
+	// Get the max count
+	int maxCount = LockManagerFactory.getIntProperty(threadContext,"org.apache.manifoldcf.ui.maxstatuscount",500000);
 	// Get the job manager handle
 	IJobManager manager = JobManagerFactory.make(threadContext);
-	JobStatus[] jobs = manager.getAllStatus();
+	JobStatus[] jobs = manager.getAllStatus(true,maxCount);
 %>
 		<table class="datatable">
 			<tr>
@@ -250,10 +252,10 @@ if (maintenanceUnderway == false)
 		}
 %>
 			</td>
-			<td class="columncell"><%="<!--jobid="+js.getJobID()+"-->"%><%=js.getDescription()%></td><td
class="columncell"><%=statusName%></td><td class="columncell"><%=startTime%></td><td
class="columncell"><%=endTime%></td>
-			<td class="columncell"><%=(js.getQueueCountExact()?"":"~")%><%=new Long(js.getDocumentsInQueue()).toString()%></td>
-			<td class="columncell"><%=(js.getOutstandingCountExact()?"":"~")%><%=new
Long(js.getDocumentsOutstanding()).toString()%></td>
-			<td class="columncell"><%=(js.getProcessedCountExact()?"":"~")%><%=new
Long(js.getDocumentsProcessed()).toString()%></td>
+			<td class="columncell"><%="<!--jobid="+js.getJobID()+"-->"%><%=org.apache.manifoldcf.ui.util.Encoder.bodyEscape(js.getDescription())%></td><td
class="columncell"><%=statusName%></td><td class="columncell"><%=startTime%></td><td
class="columncell"><%=endTime%></td>
+			<td class="columncell"><%=(js.getQueueCountExact()?"":"&gt; ")%><%=new
Long(js.getDocumentsInQueue()).toString()%></td>
+			<td class="columncell"><%=(js.getOutstandingCountExact()?"":"&gt; ")%><%=new
Long(js.getDocumentsOutstanding()).toString()%></td>
+			<td class="columncell"><%=(js.getProcessedCountExact()?"":"&gt; ")%><%=new
Long(js.getDocumentsProcessed()).toString()%></td>
 		</tr>
 <%
 	}

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1529845&r1=1529844&r2=1529845&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Mon Oct  7 14:06:14 2013
@@ -7263,113 +7263,30 @@ public class JobManager implements IJobM
       // otherwise, fire off an individual query at a time.
       if (maxCount == Integer.MAX_VALUE)
       {
+        buildCountsUsingGroupBy(whereClause,whereParams,set2Hash,set3Hash,set4Hash,set2Exact,set3Exact,set4Exact);
+      }
+      else
+      {
+        // Check if the total matching jobqueue rows exceeds the limit.  If not, we can still
use the cheaper query.
         StringBuilder sb = new StringBuilder("SELECT ");
         ArrayList list = new ArrayList();
-        
-        sb.append(JobQueue.jobIDField).append(",")
-          .append(database.constructCountClause(JobQueue.docHashField)).append(" AS doccount")
+            
+        sb.append(database.constructCountClause(JobQueue.docHashField)).append(" AS doccount")
           .append(" FROM ").append(jobQueue.getTableName()).append(" t1");
-        
-        if (whereClause != null)
-        {
-          sb.append(" WHERE EXISTS(SELECT 'x' FROM ").append(jobs.getTableName()).append("
t0 WHERE ")
-            .append(database.buildConjunctionClause(list,new ClauseDescription[]{
-              new JoinClause("t0."+Jobs.idField,"t1."+JobQueue.jobIDField)})).append(" AND
")
-            .append(whereClause)
-            .append(")");
-          list.addAll(whereParams);
-        }
-        
-        sb.append(" GROUP BY ").append(JobQueue.jobIDField);
-        
-        IResultSet set2 = database.performQuery(sb.toString(),list,null,null);
-
-        sb = new StringBuilder("SELECT ");
-        list.clear();
-        
-        sb.append(JobQueue.jobIDField).append(",")
-          .append(database.constructCountClause(JobQueue.docHashField)).append(" AS doccount")
-          .append(" FROM ").append(jobQueue.getTableName()).append(" t1 WHERE ")
-          .append(database.buildConjunctionClause(list,new ClauseDescription[]{
-            new MultiClause(JobQueue.statusField,new Object[]{
-              JobQueue.statusToString(JobQueue.STATUS_ACTIVE),
-              JobQueue.statusToString(JobQueue.STATUS_ACTIVENEEDRESCAN),
-              JobQueue.statusToString(JobQueue.STATUS_PENDING),
-              JobQueue.statusToString(JobQueue.STATUS_ACTIVEPURGATORY),
-              JobQueue.statusToString(JobQueue.STATUS_ACTIVENEEDRESCANPURGATORY),
-              JobQueue.statusToString(JobQueue.STATUS_PENDINGPURGATORY)})}));
-        if (whereClause != null)
+        addWhereClause(sb,list,whereClause,whereParams);
+        sb.append(" ").append(database.constructOffsetLimitClause(0,maxCount+1,false));
+        IResultSet countResult = database.performQuery(sb.toString(),list,null,null);
+        if (countResult.getRowCount() > 0 && ((Long)countResult.getRow(0).getValue("doccount")).longValue()
> maxCount)
         {
-          sb.append(" AND EXISTS(SELECT 'x' FROM ").append(jobs.getTableName()).append("
t0 WHERE ")
-            .append(database.buildConjunctionClause(list,new ClauseDescription[]{
-              new JoinClause("t0."+Jobs.idField,"t1."+JobQueue.jobIDField)})).append(" AND
")
-            .append(whereClause)
-            .append(")");
-          if (whereParams != null)
-            list.addAll(whereParams);
-        }
-        sb.append(" GROUP BY ").append(JobQueue.jobIDField);
-        
-        IResultSet set3 = database.performQuery(sb.toString(),list,null,null);
-
-        sb = new StringBuilder("SELECT ");
-        list.clear();
-        
-        sb.append(JobQueue.jobIDField).append(",")
-          .append(database.constructCountClause(JobQueue.docHashField)).append(" AS doccount")
-          .append(" FROM ").append(jobQueue.getTableName()).append(" t1 WHERE ")
-          .append(database.buildConjunctionClause(list,new ClauseDescription[]{
-            new MultiClause(JobQueue.statusField,new Object[]{
-              JobQueue.statusToString(JobQueue.STATUS_COMPLETE),
-              JobQueue.statusToString(JobQueue.STATUS_UNCHANGED),
-              JobQueue.statusToString(JobQueue.STATUS_PURGATORY),
-              JobQueue.statusToString(JobQueue.STATUS_ACTIVEPURGATORY),
-              JobQueue.statusToString(JobQueue.STATUS_ACTIVENEEDRESCANPURGATORY),
-              JobQueue.statusToString(JobQueue.STATUS_PENDINGPURGATORY)})}));
-        
-        if (whereClause != null)
-        {
-          sb.append(" AND EXISTS(SELECT 'x' FROM ").append(jobs.getTableName()).append("
t0 WHERE ")
-            .append(database.buildConjunctionClause(list,new ClauseDescription[]{
-              new JoinClause("t0."+Jobs.idField,"t1."+JobQueue.jobIDField)})).append(" AND
")
-            .append(whereClause)
-            .append(")");
-
-          if (whereParams != null)
-            list.addAll(whereParams);
-        }
-        
-        sb.append(" GROUP BY ").append(JobQueue.jobIDField);
-        
-        IResultSet set4 = database.performQuery(sb.toString(),list,null,null);
-        
-        for (int j = 0; j < set2.getRowCount(); j++)
-        {
-          IResultRow row = set2.getRow(j);
-          Long jobID = (Long)row.getValue(JobQueue.jobIDField);
-          set2Hash.put(jobID,(Long)row.getValue("doccount"));
-          set2Exact.put(jobID,new Boolean(true));
-        }
-        for (int j = 0; j < set3.getRowCount(); j++)
-        {
-          IResultRow row = set3.getRow(j);
-          Long jobID = (Long)row.getValue(JobQueue.jobIDField);
-          set3Hash.put(jobID,(Long)row.getValue("doccount"));
-          set3Exact.put(jobID,new Boolean(true));
+          // Too many items in queue; do it the hard way
+          buildCountsUsingIndividualQueries(whereClause,whereParams,maxCount,set2Hash,set3Hash,set4Hash,set2Exact,set3Exact,set4Exact);
         }
-        for (int j = 0; j < set4.getRowCount(); j++)
+        else
         {
-          IResultRow row = set4.getRow(j);
-          Long jobID = (Long)row.getValue(JobQueue.jobIDField);
-          set4Hash.put(jobID,(Long)row.getValue("doccount"));
-          set4Exact.put(jobID,new Boolean(true));
+          // Cheap way should still work.
+          buildCountsUsingGroupBy(whereClause,whereParams,set2Hash,set3Hash,set4Hash,set2Exact,set3Exact,set4Exact);
         }
       }
-      else
-      {
-        // Fire off an individual query with a limit for each job
-        // MHL
-      }
     }
     
     JobStatus[] rval = new JobStatus[set.getRowCount()];
@@ -7488,14 +7405,217 @@ public class JobManager implements IJobM
       rval[i] = new JobStatus(jobID.toString(),description,rstatus,((set2Value==null)?0L:set2Value.longValue()),
         ((set3Value==null)?0L:set3Value.longValue()),
         ((set4Value==null)?0L:set4Value.longValue()),
-        ((set2ExactValue==null)?false:set2ExactValue.booleanValue()),
-        ((set3ExactValue==null)?false:set3ExactValue.booleanValue()),
-        ((set4ExactValue==null)?false:set4ExactValue.booleanValue()),
+        ((set2ExactValue==null)?true:set2ExactValue.booleanValue()),
+        ((set3ExactValue==null)?true:set3ExactValue.booleanValue()),
+        ((set4ExactValue==null)?true:set4ExactValue.booleanValue()),
         startTime,endTime,errorText);
     }
     return rval;
   }
 
+  protected static ClauseDescription buildOutstandingClause()
+    throws ManifoldCFException
+  {
+    return new MultiClause(JobQueue.statusField,new Object[]{
+    JobQueue.statusToString(JobQueue.STATUS_ACTIVE),
+    JobQueue.statusToString(JobQueue.STATUS_ACTIVENEEDRESCAN),
+    JobQueue.statusToString(JobQueue.STATUS_PENDING),
+    JobQueue.statusToString(JobQueue.STATUS_ACTIVEPURGATORY),
+    JobQueue.statusToString(JobQueue.STATUS_ACTIVENEEDRESCANPURGATORY),
+    JobQueue.statusToString(JobQueue.STATUS_PENDINGPURGATORY)});
+  }
+    
+  protected static ClauseDescription buildProcessedClause()
+    throws ManifoldCFException
+  {
+    return new MultiClause(JobQueue.statusField,new Object[]{
+    JobQueue.statusToString(JobQueue.STATUS_COMPLETE),
+    JobQueue.statusToString(JobQueue.STATUS_UNCHANGED),
+    JobQueue.statusToString(JobQueue.STATUS_PURGATORY),
+    JobQueue.statusToString(JobQueue.STATUS_ACTIVEPURGATORY),
+    JobQueue.statusToString(JobQueue.STATUS_ACTIVENEEDRESCANPURGATORY),
+    JobQueue.statusToString(JobQueue.STATUS_PENDINGPURGATORY)});
+  }
+
+  protected void buildCountsUsingIndividualQueries(String whereClause, ArrayList whereParams,
int maxCount,
+    Map<Long,Long> set2Hash, Map<Long,Long> set3Hash, Map<Long,Long> set4Hash,
+    Map<Long,Boolean> set2Exact, Map<Long,Boolean> set3Exact, Map<Long,Boolean>
set4Exact)
+    throws ManifoldCFException
+  {
+    // Fire off an individual query with a limit for each job
+    
+    // First, get the list of jobs that we are interested in.
+    StringBuilder sb = new StringBuilder("SELECT ");
+    ArrayList list = new ArrayList();
+
+    sb.append(Jobs.idField).append(" FROM ").append(jobs.getTableName()).append(" t0");
+    if (whereClause != null)
+    {
+      sb.append(" WHERE ")
+        .append(whereClause);
+      if (whereParams != null)
+        list.addAll(whereParams);
+    }
+    
+    IResultSet jobSet = database.performQuery(sb.toString(),list,null,null);
+
+    // Scan the set of jobs
+    for (int i = 0; i < jobSet.getRowCount(); i++)
+    {
+      IResultRow row = jobSet.getRow(i);
+      Long jobID = (Long)row.getValue(Jobs.idField);
+      
+      // Now, for each job, fire off a separate, limited, query for each count we care about
+      sb = new StringBuilder("SELECT ");
+      list.clear();
+      sb.append(database.constructCountClause(JobQueue.docHashField)).append(" AS doccount")
+        .append(" FROM ").append(jobQueue.getTableName()).append(" WHERE ");
+      sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{new UnitaryClause(Jobs.idField,jobID)}));
+      sb.append(" ").append(database.constructOffsetLimitClause(0,maxCount+1,false));
+      
+      IResultSet totalSet = database.performQuery(sb.toString(),list,null,null);
+      if (totalSet.getRowCount() > 0)
+      {
+        long rowCount = ((Long)totalSet.getRow(0).getValue("doccount")).longValue();
+        if (rowCount > maxCount)
+        {
+          set2Hash.put(jobID,new Long(maxCount));
+          set2Exact.put(jobID,new Boolean(false));
+        }
+        else
+        {
+          set2Hash.put(jobID,new Long(rowCount));
+          set2Exact.put(jobID,new Boolean(true));
+        }
+      }
+          
+      sb = new StringBuilder("SELECT ");
+      list.clear();
+      sb.append(database.constructCountClause(JobQueue.docHashField)).append(" AS doccount")
+        .append(" FROM ").append(jobQueue.getTableName()).append(" WHERE ");
+      sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{new UnitaryClause(Jobs.idField,jobID)}));
+      sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{buildOutstandingClause()}));
+      sb.append(" ").append(database.constructOffsetLimitClause(0,maxCount+1,false));
+      
+      IResultSet outstandingSet = database.performQuery(sb.toString(),list,null,null);
+      if (outstandingSet.getRowCount() > 0)
+      {
+        long rowCount = ((Long)outstandingSet.getRow(0).getValue("doccount")).longValue();
+        if (rowCount > maxCount)
+        {
+          set3Hash.put(jobID,new Long(maxCount));
+          set3Exact.put(jobID,new Boolean(false));
+        }
+        else
+        {
+          set3Hash.put(jobID,new Long(rowCount));
+          set3Exact.put(jobID,new Boolean(true));
+        }
+      }
+
+      sb = new StringBuilder("SELECT ");
+      list.clear();
+      sb.append(database.constructCountClause(JobQueue.docHashField)).append(" AS doccount")
+        .append(" FROM ").append(jobQueue.getTableName()).append(" t1 WHERE ");
+      sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{new UnitaryClause(Jobs.idField,jobID)}));
+      sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{buildProcessedClause()}));
+      sb.append(" ").append(database.constructOffsetLimitClause(0,maxCount+1,false));
+      
+      IResultSet processedSet = database.performQuery(sb.toString(),list,null,null);
+      if (processedSet.getRowCount() > 0)
+      {
+        long rowCount = ((Long)processedSet.getRow(0).getValue("doccount")).longValue();
+        if (rowCount > maxCount)
+        {
+          set4Hash.put(jobID,new Long(maxCount));
+          set4Exact.put(jobID,new Boolean(false));
+        }
+        else
+        {
+          set4Hash.put(jobID,new Long(rowCount));
+          set4Exact.put(jobID,new Boolean(true));
+        }
+      }
+    }
+  }
+
+  protected void buildCountsUsingGroupBy(String whereClause, ArrayList whereParams,
+    Map<Long,Long> set2Hash, Map<Long,Long> set3Hash, Map<Long,Long> set4Hash,
+    Map<Long,Boolean> set2Exact, Map<Long,Boolean> set3Exact, Map<Long,Boolean>
set4Exact)
+    throws ManifoldCFException
+  {
+    StringBuilder sb = new StringBuilder("SELECT ");
+    ArrayList list = new ArrayList();
+        
+    sb.append(JobQueue.jobIDField).append(",")
+      .append(database.constructCountClause(JobQueue.docHashField)).append(" AS doccount")
+      .append(" FROM ").append(jobQueue.getTableName()).append(" t1");
+    addWhereClause(sb,list,whereClause,whereParams);
+    sb.append(" GROUP BY ").append(JobQueue.jobIDField);
+    
+    IResultSet set2 = database.performQuery(sb.toString(),list,null,null);
+
+    sb = new StringBuilder("SELECT ");
+    list.clear();
+        
+    sb.append(JobQueue.jobIDField).append(",")
+      .append(database.constructCountClause(JobQueue.docHashField)).append(" AS doccount")
+      .append(" FROM ").append(jobQueue.getTableName()).append(" t1 WHERE ")
+      .append(database.buildConjunctionClause(list,new ClauseDescription[]{buildOutstandingClause()}));
+    addWhereClause(sb,list,whereClause,whereParams);
+    sb.append(" GROUP BY ").append(JobQueue.jobIDField);
+        
+    IResultSet set3 = database.performQuery(sb.toString(),list,null,null);
+
+    sb = new StringBuilder("SELECT ");
+    list.clear();
+        
+    sb.append(JobQueue.jobIDField).append(",")
+      .append(database.constructCountClause(JobQueue.docHashField)).append(" AS doccount")
+      .append(" FROM ").append(jobQueue.getTableName()).append(" t1 WHERE ")
+      .append(database.buildConjunctionClause(list,new ClauseDescription[]{buildProcessedClause()}));
+    addWhereClause(sb,list,whereClause,whereParams);
+    sb.append(" GROUP BY ").append(JobQueue.jobIDField);
+        
+    IResultSet set4 = database.performQuery(sb.toString(),list,null,null);
+        
+    for (int j = 0; j < set2.getRowCount(); j++)
+    {
+      IResultRow row = set2.getRow(j);
+      Long jobID = (Long)row.getValue(JobQueue.jobIDField);
+      set2Hash.put(jobID,(Long)row.getValue("doccount"));
+      set2Exact.put(jobID,new Boolean(true));
+    }
+    for (int j = 0; j < set3.getRowCount(); j++)
+    {
+      IResultRow row = set3.getRow(j);
+      Long jobID = (Long)row.getValue(JobQueue.jobIDField);
+      set3Hash.put(jobID,(Long)row.getValue("doccount"));
+      set3Exact.put(jobID,new Boolean(true));
+    }
+    for (int j = 0; j < set4.getRowCount(); j++)
+    {
+      IResultRow row = set4.getRow(j);
+      Long jobID = (Long)row.getValue(JobQueue.jobIDField);
+      set4Hash.put(jobID,(Long)row.getValue("doccount"));
+      set4Exact.put(jobID,new Boolean(true));
+    }
+  }
+
+  protected void addWhereClause(StringBuilder sb, ArrayList list, String whereClause, ArrayList
whereParams)
+  {
+    if (whereClause != null)
+    {
+      sb.append(" WHERE EXISTS(SELECT 'x' FROM ").append(jobs.getTableName()).append(" t0
WHERE ")
+        .append(database.buildConjunctionClause(list,new ClauseDescription[]{
+          new JoinClause("t0."+Jobs.idField,"t1."+JobQueue.jobIDField)})).append(" AND ")
+        .append(whereClause)
+        .append(")");
+      if (whereParams != null)
+        list.addAll(whereParams);
+    }
+  }
+  
   // These methods generate reports for direct display in the UI.
 
   /** Run a 'document status' report.



Mime
View raw message