incubator-connectors-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1189308 - in /incubator/lcf/branches/CONNECTORS-284: build.xml framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Date Wed, 26 Oct 2011 16:54:57 GMT
Author: kwright
Date: Wed Oct 26 16:54:56 2011
New Revision: 1189308

URL: http://svn.apache.org/viewvc?rev=1189308&view=rev
Log:
Rearrange slow queries to meet HSQLDB requirements, and give the HSQLDB load test enough memory.

Modified:
    incubator/lcf/branches/CONNECTORS-284/build.xml
    incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java

Modified: incubator/lcf/branches/CONNECTORS-284/build.xml
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-284/build.xml?rev=1189308&r1=1189307&r2=1189308&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-284/build.xml (original)
+++ incubator/lcf/branches/CONNECTORS-284/build.xml Wed Oct 26 16:54:56 2011
@@ -2861,7 +2861,7 @@
 
     <target name="run-filesystem-loadtests-HSQLDB" depends="compile-filesystem-loadtests,calculate-filesystem-tests-condition"
if="filesystem-tests.include">
         <mkdir dir="test-HSQLDB-output/filesystem"/>
-        <junit fork="true" maxmemory="256m" dir="test-HSQLDB-output/filesystem" outputtoformatters="true"
showoutput="true" haltonfailure="true">
+        <junit fork="true" maxmemory="384m" dir="test-HSQLDB-output/filesystem" outputtoformatters="true"
showoutput="true" haltonfailure="true">
             <classpath>
                 <fileset dir="framework/lib">
                     <include name="*.jar"/>

Modified: incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1189308&r1=1189307&r2=1189308&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
(original)
+++ incubator/lcf/branches/CONNECTORS-284/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Wed Oct 26 16:54:56 2011
@@ -2061,10 +2061,10 @@ public class JobManager implements IJobM
     list.add(Jobs.statusToString(jobs.STATUS_ACTIVE));
     list.add(Jobs.statusToString(jobs.STATUS_ACTIVESEEDING));
     
-    list.add(currentTimeValue);
-    
     list.add(jobQueue.actionToString(JobQueue.ACTION_RESCAN));
     
+    list.add(currentTimeValue);
+        
     list.add(jobQueue.statusToString(JobQueue.STATUS_PENDING));
     list.add(jobQueue.statusToString(JobQueue.STATUS_PENDINGPURGATORY));
     
@@ -2073,11 +2073,9 @@ public class JobManager implements IJobM
       .append(" FROM ").append(jobQueue.getTableName())
       .append(" t0 WHERE EXISTS(SELECT 'x' FROM ").append(jobs.getTableName()).append(" t1
WHERE t0.").append(jobQueue.jobIDField)
       .append("=t1.").append(jobs.idField).append(" AND t1.").append(jobs.statusField).append("
IN(?,?)) AND ")
-      .append(jobQueue.checkTimeField).append("<=? AND (")
-      .append(jobQueue.checkActionField).append(" IS NULL OR ")
-      .append(jobQueue.checkActionField).append("=?")
-      .append(") AND (")
-      .append(jobQueue.statusField).append("=? OR ").append(jobQueue.statusField).append("=?)")
+      .append(jobQueue.checkActionField).append("=? AND ")
+      .append(jobQueue.checkTimeField).append("<=? AND ")
+      .append(jobQueue.statusField).append(" IN(?,?)")
       .append(" ORDER BY ").append(jobQueue.docPriorityField).append(" ASC ").append(database.constructOffsetLimitClause(0,1));
 
 
@@ -2091,22 +2089,44 @@ public class JobManager implements IJobM
     return rval;
   }
 
-  protected void addDocumentCriteria(StringBuilder sb, ArrayList list, Long currentTimeValue,
Long currentPriorityValue)
+  /** Fetch and process documents matching the passed-in criteria */
+  protected void fetchAndProcessDocuments(ArrayList answers, Long currentTimeValue, Long
currentPriorityValue,
+    ThrottleLimit vList, IRepositoryConnection[] connections)
     throws ManifoldCFException
   {
+
+    // Note well: This query does not do "FOR UPDATE".  The reason is that only one thread
can possibly change the document's state to active.
+    // When FOR UPDATE was included, deadlock conditions were common because of the complexity
of this query.
+
+    ArrayList list = new ArrayList();
+
+    StringBuilder sb = new StringBuilder("SELECT t0.");
+    sb.append(jobQueue.idField).append(",t0.");
+    if (Logging.scheduling.isDebugEnabled())
+      sb.append(jobQueue.docPriorityField).append(",t0.");
+    sb.append(jobQueue.jobIDField).append(",t0.");
+    sb.append(jobQueue.docHashField).append(",t0.");
+    sb.append(jobQueue.docIDField).append(",t0.");
+    sb.append(jobQueue.statusField).append(",t0.");
+    sb.append(jobQueue.failTimeField).append(",t0.");
+    sb.append(jobQueue.failCountField).append(",t0.");
+    sb.append(jobQueue.prioritySetField).append(" FROM ").append(jobQueue.getTableName())
+      .append(" t0 WHERE EXISTS(SELECT 'x' FROM ").append(jobs.getTableName()).append(" t1
WHERE t0.").append(jobQueue.jobIDField)
+      .append("=t1.").append(jobs.idField).append(" AND t1.").append(jobs.statusField).append("
IN(?,?) AND t1.")
+      .append(jobs.priorityField).append("=?) AND ");
+
+    list.add(jobs.statusToString(jobs.STATUS_ACTIVE));
+    list.add(jobs.statusToString(jobs.STATUS_ACTIVESEEDING));
     
-    list.add(currentTimeValue);
-    
+    list.add(currentPriorityValue);
+
     list.add(jobQueue.actionToString(JobQueue.ACTION_RESCAN));
+
+    list.add(currentTimeValue);
     
     list.add(jobQueue.statusToString(JobQueue.STATUS_PENDING));
     list.add(jobQueue.statusToString(JobQueue.STATUS_PENDINGPURGATORY));
     
-    list.add(jobs.statusToString(jobs.STATUS_ACTIVE));
-    list.add(jobs.statusToString(jobs.STATUS_ACTIVESEEDING));
-    
-    list.add(currentPriorityValue);
-
     list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVE));
     list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY));
     list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCAN));
@@ -2114,12 +2134,10 @@ public class JobManager implements IJobM
     list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED));
     list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED));
     
-    sb.append("t0.").append(jobQueue.checkTimeField).append("<=? AND ");
-    sb.append("(t0.").append(jobQueue.checkActionField).append(" IS NULL OR t0.").append(jobQueue.checkActionField)
-      .append("=?) AND ");
-    sb.append("(t0.").append(jobQueue.statusField).append("=? OR t0.").append(jobQueue.statusField).append("=?)
AND t0.").append(jobQueue.jobIDField).append("=t1.").append(jobs.idField).append(" AND t1.")
-      .append(jobs.statusField).append(" IN (?,?) AND t1.")
-      .append(jobs.priorityField).append("=? AND ");
+    sb.append("t0.").append(jobQueue.checkActionField).append("=? AND ")
+      .append("t0.").append(jobQueue.checkTimeField).append("<=? AND ")
+      .append("t0.").append(jobQueue.statusField).append(" IN (?,?) AND ");
+      
     sb.append("NOT EXISTS(SELECT 'x' FROM ").append(jobQueue.getTableName()).append(" t2
WHERE t0.")
       .append(jobQueue.docHashField).append("=t2.").append(jobQueue.docHashField).append("
AND t0.")
       .append(jobQueue.jobIDField).append("!=t2.").append(jobQueue.jobIDField).append(" AND
t2.")
@@ -2130,33 +2148,6 @@ public class JobManager implements IJobM
       .append(jobQueue.idField).append("=t3.").append(jobQueue.prereqEventManager.ownerField).append("
AND t3.")
       .append(jobQueue.prereqEventManager.eventNameField).append("=t4.").append(eventManager.eventNameField)
       .append(")");
-  }
-
-  /** Fetch and process documents matching the passed-in criteria */
-  protected void fetchAndProcessDocuments(ArrayList answers, Long currentTimeValue, Long
currentPriorityValue,
-    ThrottleLimit vList, IRepositoryConnection[] connections)
-    throws ManifoldCFException
-  {
-
-    // Note well: This query does not do "FOR UPDATE".  The reason is that only one thread
can possibly change the document's state to active.
-    // When FOR UPDATE was included, deadlock conditions were common because of the complexity
of this query.
-
-    ArrayList list = new ArrayList();
-
-    StringBuilder sb = new StringBuilder("SELECT t0.");
-    sb.append(jobQueue.idField).append(",t0.");
-    if (Logging.scheduling.isDebugEnabled())
-      sb.append(jobQueue.docPriorityField).append(",t0.");
-    sb.append(jobQueue.jobIDField).append(",t0.");
-    sb.append(jobQueue.docHashField).append(",t0.");
-    sb.append(jobQueue.docIDField).append(",t0.");
-    sb.append(jobQueue.statusField).append(",t0.");
-    sb.append(jobQueue.failTimeField).append(",t0.");
-    sb.append(jobQueue.failCountField).append(",t0.");
-    sb.append(jobQueue.prioritySetField).append(" FROM ").append(jobQueue.getTableName()).append("
t0,")
-      .append(jobs.getTableName()).append(" t1 WHERE ");
-
-    addDocumentCriteria(sb,list,currentTimeValue,currentPriorityValue);
 
     sb.append(" ORDER BY t0.").append(jobQueue.docPriorityField).append(" ASC ");
 
@@ -2584,12 +2575,16 @@ public class JobManager implements IJobM
   {
     // The query here mirrors the carrydown.restoreRecords() delete query!  However, it also
fetches enough information to build a DocumentDescription
     // object for return, and so a join is necessary against the jobqueue table.
+    ArrayList newlist = new ArrayList();
+    newlist.add(jobID);
+    newlist.addAll(list);
     String query = "SELECT t0."+jobQueue.idField+",t0."+jobQueue.docHashField+",t0."+jobQueue.docIDField+"
FROM "+
-      jobQueue.getTableName()+" t0 WHERE EXISTS(SELECT 'x' FROM "+carryDown.getTableName()+
-      " t1 WHERE t1."+carryDown.parentIDHashField+" IN ("+queryPart+") AND t1."+carryDown.childIDHashField+"=t0."+jobQueue.docHashField+
-      " AND t0."+jobQueue.jobIDField+"=t1."+carryDown.jobIDField+") AND t0."+jobQueue.jobIDField+"=?";
-    list.add(jobID);
-    IResultSet set = database.performQuery(query,list,null,null);
+      jobQueue.getTableName()+" t0 WHERE "+
+      "t0."+jobQueue.jobIDField+"=? AND "+
+      "EXISTS(SELECT 'x' FROM "+carryDown.getTableName()+" t1 WHERE "+
+      "t0."+jobQueue.jobIDField+"=t1."+carryDown.jobIDField+" AND "+
+      "t1."+carryDown.parentIDHashField+" IN ("+queryPart+") AND t1."+carryDown.childIDHashField+"=t0."+jobQueue.docHashField+")";
+    IResultSet set = database.performQuery(query,newlist,null,null);
     int i = 0;
     while (i < set.getRowCount())
     {
@@ -4046,15 +4041,21 @@ public class JobManager implements IJobM
   {
     // The query here mirrors the carrydown.restoreRecords() delete query!  However, it also
fetches enough information to build a DocumentDescription
     // object for return, and so a join is necessary against the jobqueue table.
-    //???
+    ArrayList newlist = new ArrayList();
+    newlist.add(jobID);
+    newlist.addAll(list);
+    newlist.add(carryDown.statusToString(carryDown.ISNEW_BASE));
+
     String query = "SELECT t0."+jobQueue.idField+",t0."+jobQueue.docHashField+",t0."+jobQueue.docIDField+"
FROM "+
-      jobQueue.getTableName()+" t0 WHERE EXISTS(SELECT 'x' FROM "+carryDown.getTableName()+
-      " t1 WHERE "+carryDown.parentIDHashField+" IN ("+queryPart+") AND t1."+carryDown.childIDHashField+"=t0."+jobQueue.docHashField+
-      " AND t0."+jobQueue.jobIDField+"=t1."+jobQueue.jobIDField+" AND t1."+carryDown.newField+"=?)
AND t0."+jobQueue.jobIDField+"=?";
-    list.add(carryDown.statusToString(carryDown.ISNEW_BASE));
-    list.add(jobID);
+      jobQueue.getTableName()+" t0 WHERE "+
+      "t0."+jobQueue.jobIDField+"=? AND "+
+      "EXISTS(SELECT 'x' FROM "+carryDown.getTableName()+" t1 WHERE "+
+      "t0."+jobQueue.jobIDField+"=t1."+jobQueue.jobIDField+" AND "+
+      carryDown.parentIDHashField+" IN ("+queryPart+") AND "+
+      "t1."+carryDown.childIDHashField+"=t0."+jobQueue.docHashField+" AND "+
+      "t1."+carryDown.newField+"=?)";
 
-    IResultSet set = database.performQuery(query,list,null,null);
+    IResultSet set = database.performQuery(query,newlist,null,null);
     int i = 0;
     while (i < set.getRowCount())
     {
@@ -6904,22 +6905,22 @@ public class JobManager implements IJobM
         sb.append(fieldPrefix).append(jobQueue.statusField).append(" IN (?,?)");
         break;
       case DOCSTATUS_PROCESSING:
+        list.add(jobQueue.actionToString(jobQueue.ACTION_RESCAN));
         list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVE));
         list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCAN));
         list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY));
         list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY));
-        list.add(jobQueue.actionToString(jobQueue.ACTION_RESCAN));
-        sb.append(fieldPrefix).append(jobQueue.statusField).append(" IN (?,?,?,?)")
-          .append(" AND (").append(fieldPrefix).append(jobQueue.checkActionField).append("
IS NULL OR ").append(fieldPrefix).append(jobQueue.checkActionField).append("=?)");
+        sb.append("(").append(fieldPrefix).append(jobQueue.checkActionField).append(" IS
NULL OR ").append(fieldPrefix).append(jobQueue.checkActionField).append("=?)")
+          .append(" AND ").append(fieldPrefix).append(jobQueue.statusField).append(" IN (?,?,?,?)");
         break;
       case DOCSTATUS_EXPIRING:
+        list.add(jobQueue.actionToString(jobQueue.ACTION_REMOVE));
         list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVE));
         list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCAN));
         list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY));
         list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY));
-        list.add(jobQueue.actionToString(jobQueue.ACTION_REMOVE));
-        sb.append(fieldPrefix).append(jobQueue.statusField).append(" IN (?,?,?,?)")
-          .append(" AND ").append(fieldPrefix).append(jobQueue.checkActionField).append("=?");
+        sb.append(fieldPrefix).append(jobQueue.checkActionField).append("=?")
+          .append(" AND ").append(fieldPrefix).append(jobQueue.statusField).append(" IN (?,?,?,?)");
         break;
       case DOCSTATUS_DELETING:
         list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED));
@@ -6928,36 +6929,36 @@ public class JobManager implements IJobM
         sb.append(fieldPrefix).append(jobQueue.statusField).append(" IN (?,?,?)");
         break;
       case DOCSTATUS_READYFORPROCESSING:
+        list.add(jobQueue.actionToString(jobQueue.ACTION_RESCAN));
         list.add(jobQueue.statusToString(jobQueue.STATUS_PENDING));
         list.add(jobQueue.statusToString(jobQueue.STATUS_PENDINGPURGATORY));
-        list.add(jobQueue.actionToString(jobQueue.ACTION_RESCAN));
-        sb.append(fieldPrefix).append(jobQueue.statusField).append(" IN (?,?)")
-          .append(" AND (").append(fieldPrefix).append(jobQueue.checkActionField).append("
IS NULL OR ").append(fieldPrefix).append(jobQueue.checkActionField).append("=?)")
-          .append(" AND ").append(fieldPrefix).append(jobQueue.checkTimeField).append("<=").append(nowTime.toString());
+        sb.append("(").append(fieldPrefix).append(jobQueue.checkActionField).append(" IS
NULL OR ").append(fieldPrefix).append(jobQueue.checkActionField).append("=?)")
+          .append(" AND ").append(fieldPrefix).append(jobQueue.checkTimeField).append("<=").append(nowTime.toString())
+          .append(" AND ").append(fieldPrefix).append(jobQueue.statusField).append(" IN (?,?)");
         break;
       case DOCSTATUS_READYFOREXPIRATION:
+        list.add(jobQueue.actionToString(jobQueue.ACTION_REMOVE));
         list.add(jobQueue.statusToString(jobQueue.STATUS_PENDING));
         list.add(jobQueue.statusToString(jobQueue.STATUS_PENDINGPURGATORY));
-        list.add(jobQueue.actionToString(jobQueue.ACTION_REMOVE));
-        sb.append(fieldPrefix).append(jobQueue.statusField).append(" IN (?,?)")
-          .append(" AND ").append(fieldPrefix).append(jobQueue.checkActionField).append("=?")
-          .append(" AND ").append(fieldPrefix).append(jobQueue.checkTimeField).append("<=").append(nowTime.toString());
+        sb.append(fieldPrefix).append(jobQueue.checkActionField).append("=?")
+          .append(" AND ").append(fieldPrefix).append(jobQueue.checkTimeField).append("<=").append(nowTime.toString())
+          .append(" AND ").append(fieldPrefix).append(jobQueue.statusField).append(" IN (?,?)");
         break;
       case DOCSTATUS_WAITINGFORPROCESSING:
+        list.add(jobQueue.actionToString(jobQueue.ACTION_RESCAN));
         list.add(jobQueue.statusToString(jobQueue.STATUS_PENDING));
         list.add(jobQueue.statusToString(jobQueue.STATUS_PENDINGPURGATORY));
-        list.add(jobQueue.actionToString(jobQueue.ACTION_RESCAN));
-        sb.append(fieldPrefix).append(jobQueue.statusField).append(" IN (?,?)")
-          .append(" AND (").append(fieldPrefix).append(jobQueue.checkActionField).append("
IS NULL OR ").append(fieldPrefix).append(jobQueue.checkActionField).append("=?)")
-          .append(" AND ").append(fieldPrefix).append(jobQueue.checkTimeField).append(">").append(nowTime.toString());
+          sb.append("(").append(fieldPrefix).append(jobQueue.checkActionField).append(" IS
NULL OR ").append(fieldPrefix).append(jobQueue.checkActionField).append("=?)")
+          .append(" AND ").append(fieldPrefix).append(jobQueue.checkTimeField).append(">").append(nowTime.toString())
+          .append(" AND ").append(fieldPrefix).append(jobQueue.statusField).append(" IN (?,?)");
         break;
       case DOCSTATUS_WAITINGFOREXPIRATION:
+        list.add(jobQueue.actionToString(jobQueue.ACTION_REMOVE));
         list.add(jobQueue.statusToString(jobQueue.STATUS_PENDING));
         list.add(jobQueue.statusToString(jobQueue.STATUS_PENDINGPURGATORY));
-        list.add(jobQueue.actionToString(jobQueue.ACTION_REMOVE));
-        sb.append(fieldPrefix).append(jobQueue.statusField).append(" IN (?,?)")
-          .append(" AND ").append(fieldPrefix).append(jobQueue.checkActionField).append("=?")
-          .append(" AND ").append(fieldPrefix).append(jobQueue.checkTimeField).append(">").append(nowTime.toString());
+          sb.append(fieldPrefix).append(jobQueue.checkActionField).append("=?")
+          .append(" AND ").append(fieldPrefix).append(jobQueue.checkTimeField).append(">").append(nowTime.toString())
+          .append(" AND ").append(fieldPrefix).append(jobQueue.statusField).append(" IN (?,?)");
         break;
       case DOCSTATUS_WAITINGFOREVER:
         list.add(jobQueue.statusToString(jobQueue.STATUS_PENDING));



Mime
View raw message