incubator-connectors-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1082484 - in /incubator/lcf/trunk: ./ framework/core/src/main/java/org/apache/manifoldcf/core/database/ framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/
Date Thu, 17 Mar 2011 13:30:27 GMT
Author: kwright
Date: Thu Mar 17 13:30:26 2011
New Revision: 1082484

URL: http://svn.apache.org/viewvc?rev=1082484&view=rev
Log:
Fix for CONNECTORS-169. Add a method to database abstraction to handle maximum OR clauses.

Modified:
    incubator/lcf/trunk/CHANGES.txt
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java

Modified: incubator/lcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/CHANGES.txt?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/CHANGES.txt (original)
+++ incubator/lcf/trunk/CHANGES.txt Thu Mar 17 13:30:26 2011
@@ -3,6 +3,11 @@ $Id$
 
 ==================  0.2-dev ==================
 
+CONNECTORS-169: Add a method to the database abstraction to
+return the maximum number of OR clauses in a query.  This is to make
+Derby efficient, since it can't seem to use indexes in this situation
+(Karl Wright)
+
 CONNECTORS-166: Pull a Derby fix down from trunk into Derby 10.7.1.1,
 to fix deadlock problems
 (Oleg Tikhonov, Karl Wright)

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
Thu Mar 17 13:30:26 2011
@@ -357,6 +357,17 @@ public class BaseTable
     return dbInterface.getMaxInClause();
   }
 
+  /** Obtain the maximum number of individual clauses that should be
+  * present in a sequence of OR clauses.  Exceeding this amount will potentially cause the
query performance
+  * to drop.
+  *@return the maximum number of OR clause members.
+  */
+  protected int getMaxOrClause()
+  {
+    return dbInterface.getMaxOrClause();
+  }
+
+
   /** Set up a base object from a database row.
   *@param object is the object to read into.
   *@param resultRow is the row to use to initialize the object.

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
Thu Mar 17 13:30:26 2011
@@ -964,6 +964,17 @@ public class DBInterfaceDerby extends Da
     return 100;
   }
 
+  /** Obtain the maximum number of individual clauses that should be
+  * present in a sequence of OR clauses.  Exceeding this amount will potentially cause the
query performance
+  * to drop.
+  *@return the maximum number of OR clause members.
+  */
+  public int getMaxOrClause()
+  {
+    return 1;
+  }
+
+
   /** Begin a database transaction.  This method call MUST be paired with an endTransaction()
call,
   * or database handles will be lost.  If the transaction should be rolled back, then signalRollback()
should
   * be called before the transaction is ended.

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
Thu Mar 17 13:30:26 2011
@@ -936,6 +936,17 @@ public class DBInterfaceHSQLDB extends D
     return 100;
   }
 
+  /** Obtain the maximum number of individual clauses that should be
+  * present in a sequence of OR clauses.  Exceeding this amount will potentially cause the
query performance
+  * to drop.
+  *@return the maximum number of OR clause members.
+  */
+  public int getMaxOrClause()
+  {
+    return 25;
+  }
+
+
   /** Begin a database transaction.  This method call MUST be paired with an endTransaction()
call,
   * or database handles will be lost.  If the transaction should be rolled back, then signalRollback()
should
   * be called before the transaction is ended.

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
Thu Mar 17 13:30:26 2011
@@ -633,6 +633,17 @@ public class DBInterfaceMySQL extends Da
     return 100;
   }
 
+  /** Obtain the maximum number of individual clauses that should be
+  * present in a sequence of OR clauses.  Exceeding this amount will potentially cause the
query performance
+  * to drop.
+  *@return the maximum number of OR clause members.
+  */
+  public int getMaxOrClause()
+  {
+    return 25;
+  }
+
+
   /** Begin a database transaction.  This method call MUST be paired with an endTransaction()
call,
   * or database handles will be lost.  If the transaction should be rolled back, then signalRollback()
should
   * be called before the transaction is ended.

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
Thu Mar 17 13:30:26 2011
@@ -988,6 +988,17 @@ public class DBInterfacePostgreSQL exten
     return 100;
   }
 
+  /** Obtain the maximum number of individual clauses that should be
+  * present in a sequence of OR clauses.  Exceeding this amount will potentially cause the
query performance
+  * to drop.
+  *@return the maximum number of OR clause members.
+  */
+  public int getMaxOrClause()
+  {
+    return 25;
+  }
+
+
   /** Begin a database transaction.  This method call MUST be paired with an endTransaction()
call,
   * or database handles will be lost.  If the transaction should be rolled back, then signalRollback()
should
   * be called before the transaction is ended.

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
Thu Mar 17 13:30:26 2011
@@ -304,6 +304,13 @@ public interface IDBInterface
   */
   public int getMaxInClause();
 
+  /** Obtain the maximum number of individual clauses that should be
+  * present in a sequence of OR clauses.  Exceeding this amount will potentially cause the
query performance
+  * to drop.
+  *@return the maximum number of OR clause members.
+  */
+  public int getMaxOrClause();
+
   /** Begin a database transaction.  This method call MUST be paired with an endTransaction()
call,
   * or database handles will be lost.  If the transaction should be rolled back, then signalRollback()
should
   * be called before the transaction is ended.

Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java
(original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java
Thu Mar 17 13:30:26 2011
@@ -227,7 +227,7 @@ public class Carrydown extends org.apach
     HashMap duplicateRemoval = new HashMap();
     HashMap presentMap = new HashMap();
 
-    int maxClause = 25;
+    int maxClause = getMaxOrClause();
     StringBuffer sb = new StringBuffer();
     ArrayList list = new ArrayList();
     int i = 0;
@@ -409,7 +409,7 @@ public class Carrydown extends org.apach
     beginTransaction();
     try
     {
-      int maxClause = 25;
+      int maxClause = getMaxOrClause();
       StringBuffer sb = new StringBuffer();
       ArrayList list = new ArrayList();
       int i = 0;
@@ -481,7 +481,7 @@ public class Carrydown extends org.apach
     beginTransaction();
     try
     {
-      int maxClause = 25;
+      int maxClause = getMaxOrClause();
       StringBuffer sb = new StringBuffer();
       StringBuffer sb2 = new StringBuffer();
       ArrayList list = new ArrayList();

Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
(original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
Thu Mar 17 13:30:26 2011
@@ -607,7 +607,7 @@ public class HopCount extends org.apache
       i++;
     }
 
-    int maxClause = 25;
+    int maxClause = getMaxOrClause();
     i = 0;
     int k = 0;
     while (i < parentIdentifierHashes.length)
@@ -809,7 +809,7 @@ public class HopCount extends org.apache
       // an IN clause for one index field, and = clauses for the others.  So I have to generate
this
       // as ORed together match tuples.  I do 25 at a pop, which is arbitrary.
 
-      int maxClause = 25;
+      int maxClause = getMaxOrClause();
       StringBuffer sb = new StringBuffer();
       ArrayList list = new ArrayList();
       int i = 0;
@@ -1111,7 +1111,7 @@ public class HopCount extends org.apache
         //       ...
         //       OR (t0.jobid=? AND t0.childidhash=? AND t0.childid=?))
 
-        int maxClause = 25;
+        int maxClause = getMaxOrClause();
         ArrayList list = new ArrayList();
         StringBuffer sb = new StringBuffer();
         int i = 0;
@@ -1305,7 +1305,7 @@ public class HopCount extends org.apache
       // Accumulate the ids of rows where I need deps too.  This is keyed by id and has the
right answer object as a value.
       Map depsMap = new HashMap();
 
-      int maxClause = 25;
+      int maxClause = getMaxOrClause();
       i = 0;
       int k = 0;
       while (i < unansweredQuestions.length)
@@ -2854,7 +2854,7 @@ public class HopCount extends org.apache
 
       HashMap referenceMap = new HashMap();
 
-      int maxClause = 25;
+      int maxClause = getMaxOrClause();
       StringBuffer sb = new StringBuffer();
       ArrayList list = new ArrayList();
       k = 0;

Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java
(original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java
Thu Mar 17 13:30:26 2011
@@ -190,7 +190,7 @@ public class IntrinsicLink extends org.a
       // one thread can make it into here at any given time for any given parent.
       //performLock();
       HashMap duplicateRemoval = new HashMap();
-      int maxClause = 25;
+      int maxClause = getMaxOrClause();
       StringBuffer sb = new StringBuffer();
       ArrayList list = new ArrayList();
       int i = 0;
@@ -319,7 +319,7 @@ public class IntrinsicLink extends org.a
     {
       if (sourceDocumentIDHashes != null)
       {
-        int maxClause = 25;
+        int maxClause = getMaxOrClause();
         StringBuffer sb = new StringBuffer();
         ArrayList list = new ArrayList();
         int i = 0;
@@ -401,7 +401,7 @@ public class IntrinsicLink extends org.a
     beginTransaction();
     try
     {
-      int maxClause = 25;
+      int maxClause = getMaxOrClause();
       StringBuffer sb = new StringBuffer();
       ArrayList list = new ArrayList();
       int i = 0;

Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1082484&r1=1082483&r2=1082484&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
(original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Thu Mar 17 13:30:26 2011
@@ -2541,7 +2541,7 @@ public class JobManager implements IJobM
     HashMap resultHash = new HashMap();
     StringBuffer sb = new StringBuffer();
     ArrayList list = new ArrayList();
-    int maxCount = 25;
+    int maxCount = database.getMaxOrClause();
     int i = 0;
     int z = 0;
     while (i < docIDHashes.length)
@@ -4003,7 +4003,7 @@ public class JobManager implements IJobM
     HashMap resultHash = new HashMap();
     StringBuffer sb = new StringBuffer();
     ArrayList list = new ArrayList();
-    int maxCount = 25;
+    int maxCount = database.getMaxOrClause();
     int i = 0;
     int z = 0;
     while (i < parentIDHashes.length)



Mime
View raw message