incubator-connectors-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1197576 - in /incubator/lcf/trunk/framework: core/src/test/java/org/apache/manifoldcf/core/tests/ pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/
Date Fri, 04 Nov 2011 14:02:55 GMT
Author: kwright
Date: Fri Nov  4 14:02:54 2011
New Revision: 1197576

URL: http://svn.apache.org/viewvc?rev=1197576&view=rev
Log:
Optimize for postgresql; still trying to figure out how to make all databases happily read
from order-by index

Modified:
    incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BasePostgresql.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java

Modified: incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BasePostgresql.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BasePostgresql.java?rev=1197576&r1=1197575&r2=1197576&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BasePostgresql.java
(original)
+++ incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BasePostgresql.java
Fri Nov  4 14:02:54 2011
@@ -107,6 +107,7 @@ public class BasePostgresql
       "  <property name=\"org.apache.manifoldcf.crawler.cleanupthreads\" value=\"10\"/>\n"
+
       "  <property name=\"org.apache.manifoldcf.crawler.deletethreads\" value=\"10\"/>\n"
+
       "  <property name=\"org.apache.manifoldcf.database.maxhandles\" value=\"80\"/>\n"
+
+      "  <property name=\"org.apache.manifoldcf.database.maxquerytime\" value=\"15\"/>\n"
+
       "  <property name=\"org.apache.manifoldcf.logconfigfile\" value=\""+loggingFile.getAbsolutePath().replaceAll("\\\\","/")+"\"/>\n"
+
       "</configuration>\n");
 

Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1197576&r1=1197575&r2=1197576&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
(original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Fri Nov  4 14:02:54 2011
@@ -2094,6 +2094,7 @@ public class JobManager implements IJobM
       .append(" t0 WHERE ");
       
     sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{
+      //new UnitaryClause(jobQueue.docPriorityField,">=",new Long(0L)),
       new MultiClause(jobQueue.statusField,
         new Object[]{jobQueue.statusToString(JobQueue.STATUS_PENDING),
           jobQueue.statusToString(JobQueue.STATUS_PENDINGPURGATORY)}),
@@ -2108,7 +2109,12 @@ public class JobManager implements IJobM
         new JoinClause("t1."+jobs.idField,"t0."+jobQueue.jobIDField)}))
       .append(") ");
       
-    sb.append(" ORDER BY ").append(jobQueue.docPriorityField).append(" ASC ").append(database.constructOffsetLimitClause(0,1));
+    sb.append(" ORDER BY ")
+      .append(jobQueue.docPriorityField).append(" ASC,")
+      .append(jobQueue.statusField).append(" ASC,")
+      .append(jobQueue.checkActionField).append(" ASC,")
+      .append(jobQueue.checkTimeField).append(" ASC ")
+      .append(database.constructOffsetLimitClause(0,1));
 
     IResultSet set = database.performQuery(sb.toString(),list,null,null,1,null);
     if (set.getRowCount() > 0)
@@ -2144,6 +2150,7 @@ public class JobManager implements IJobM
       .append(jobQueue.prioritySetField).append(" FROM ").append(jobQueue.getTableName()).append("
t0 WHERE ");
     
     sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{
+      //new UnitaryClause("t0."+jobQueue.docPriorityField,">=",new Long(0L)),
       new MultiClause("t0."+jobQueue.statusField,new Object[]{
         jobQueue.statusToString(JobQueue.STATUS_PENDING),
         jobQueue.statusToString(JobQueue.STATUS_PENDINGPURGATORY)}),
@@ -2179,7 +2186,12 @@ public class JobManager implements IJobM
       .append(jobQueue.prereqEventManager.eventNameField).append("=t4.").append(eventManager.eventNameField)
       .append(")");
 
-    sb.append(" ORDER BY t0.").append(jobQueue.docPriorityField).append(" ASC ");
+    sb.append(" ORDER BY ")
+      .append("t0.").append(jobQueue.docPriorityField).append(" ASC,")
+      .append("t0.").append(jobQueue.statusField).append(" ASC,")
+      .append("t0.").append(jobQueue.checkActionField).append(" ASC,")
+      .append("t0.").append(jobQueue.checkTimeField).append(" ASC ");
+
 
     // Before entering the transaction, we must provide the throttlelimit object with all
the connector
     // instances it could possibly need.  The purpose for doing this is to prevent a deadlock
where

Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java?rev=1197576&r1=1197575&r2=1197576&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
(original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
Fri Nov  4 14:02:54 2011
@@ -208,7 +208,7 @@ public class JobQueue extends org.apache
       IndexDescription actionTimeStatusIndex = new IndexDescription(false,new String[]{statusField,checkActionField,checkTimeField});
       IndexDescription prioritysetStatusIndex = new IndexDescription(false,new String[]{statusField,prioritySetField});
       // No evidence that the extra fields help at all, for any database...
-      IndexDescription docpriorityIndex = new IndexDescription(false,new String[]{docPriorityField/*,statusField,checkActionField,checkTimeField*/});
+      IndexDescription docpriorityIndex = new IndexDescription(false,new String[]{docPriorityField,statusField,checkActionField,checkTimeField});
 
       // Get rid of unused indexes
       Map indexes = getTableIndexes(null,null);



Mime
View raw message