manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1436686 - in /manifoldcf/trunk: CHANGES.txt framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
Date Mon, 21 Jan 2013 23:18:32 GMT
Author: kwright
Date: Mon Jan 21 23:18:32 2013
New Revision: 1436686

URL: http://svn.apache.org/viewvc?rev=1436686&view=rev
Log:
Fix for CONNECTORS-618.

Modified:
    manifoldcf/trunk/CHANGES.txt
    manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
    manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java

Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1436686&r1=1436685&r2=1436686&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Mon Jan 21 23:18:32 2013
@@ -8,6 +8,11 @@ CONNECTORS-617: Simple report NPE under 
 
 ======================= Release 1.1 =====================
 
+CONNECTORS-618: MySQL orders indexes so that NULL values are first.
+This is a problem for the stuffer query, which then must go through
+millions of rows before it finds the one it is looking for.
+(Shigeki Kobayashi, Karl Wright)
+
 CONNECTORS-616: Work around Solr 4.0 or Jetty bug where connections
 are dropped randomly under multithreaded load.  Broken pipe exceptions
 are now retried after a minute, for up to three times, before the Solr

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1436686&r1=1436685&r2=1436686&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Mon Jan 21 23:18:32 2013
@@ -2132,7 +2132,7 @@ public class JobManager implements IJobM
     {
       IResultRow row = set.getRow(0);
       Double docPriority = (Double)row.getValue(jobQueue.docPriorityField);
-      if (docPriority != null)
+      if (docPriority != null && docPriority.doubleValue() < jobQueue.noDocPriorityValue)
         scanRecord.addBins(docPriority);
     }
     return rval;

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java?rev=1436686&r1=1436685&r2=1436686&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
Mon Jan 21 23:18:32 2013
@@ -110,6 +110,9 @@ public class JobQueue extends org.apache
   public static final String prioritySetField = "priorityset";
   public static final String checkActionField = "checkaction";
 
+  public static final double noDocPriorityValue = 1e9;
+  public static final Double nullDocPriority = new Double(noDocPriorityValue + 1.0);
+  
   protected static Map statusMap;
 
   static
@@ -198,7 +201,10 @@ public class JobQueue extends org.apache
       }
       else
       {
-        // Upgrade code goes here, if needed
+        // Upgrade; null docpriority fields bashed to 'infinity', so they don't slow down
MySQL
+        Map map = new HashMap();
+        map.put(docPriorityField,nullDocPriority);
+        performUpdate(map,"WHERE "+docPriorityField+" IS NULL",null,null);
       }
 
       // Secondary table installation
@@ -688,7 +694,7 @@ public class JobQueue extends org.apache
   {
     HashMap map = new HashMap();
     map.put(prioritySetField,null);
-    map.put(docPriorityField,null);
+    map.put(docPriorityField,nullDocPriority);
     ArrayList list = new ArrayList();
     String query = buildConjunctionClause(list,new ClauseDescription[]{
       new UnitaryClause(jobIDField,jobID)});
@@ -715,7 +721,7 @@ public class JobQueue extends org.apache
       actionFieldValue = null;
       checkTimeValue = null;
       // Remove document priority; we don't want to pollute the queue.  See CONNECTORS-290.
-      map.put(docPriorityField,null);
+      map.put(docPriorityField,nullDocPriority);
       map.put(prioritySetField,null);
       break;
     case STATUS_ACTIVENEEDRESCAN:



Mime
View raw message