manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1210307 - in /incubator/lcf/trunk: ./ framework/core/src/main/java/org/apache/manifoldcf/core/database/ framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jo...
Date Mon, 05 Dec 2011 01:57:44 GMT
Author: kwright
Date: Mon Dec  5 01:57:43 2011
New Revision: 1210307

URL: http://svn.apache.org/viewvc?rev=1210307&view=rev
Log:
Fix for CONNECTORS-284.  Upgrade to a new version of HSQLDB, which includes the hint 'USING
INDEX' to tell the planner to use the order-by index if it can.  This makes HSQLDB now scalable
as far as ManifoldCF is concerned.

Modified:
    incubator/lcf/trunk/CHANGES.txt
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
    incubator/lcf/trunk/lib/hsqldb.jar
    incubator/lcf/trunk/mvn-bootstrap.bat
    incubator/lcf/trunk/mvn-bootstrap.sh
    incubator/lcf/trunk/pom.xml
    incubator/lcf/trunk/site/src/documentation/content/xdocs/how-to-build-and-deploy.xml

Modified: incubator/lcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/CHANGES.txt?rev=1210307&r1=1210306&r2=1210307&view=diff
==============================================================================
--- incubator/lcf/trunk/CHANGES.txt (original)
+++ incubator/lcf/trunk/CHANGES.txt Mon Dec  5 01:57:43 2011
@@ -3,6 +3,12 @@ $Id$
 
 ======================= 0.4-dev =====================
 
+CONNECTORS-284: HSQLDB runs out of memory on an extended load
+test, and had other problems too.  A new version of the db software was
+needed to correct this issue.  Thanks to the HSQLDB team for their
+assistance!
+(Fred Toussi, Karl Wright)
+
 CONNECTORS-299: Handle Postgresql 9.1's newfound ability to throw
 deadlock conditions on transaction ends.
 (Erlend GarĂ¥sen, Karl Wright)

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java?rev=1210307&r1=1210306&r2=1210307&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
Mon Dec  5 01:57:43 2011
@@ -1075,9 +1075,10 @@ public class DBInterfaceDerby extends Da
   * This method constructs an offset/limit clause in the proper manner for the database in
question.
   *@param offset is the starting offset number.
   *@param limit is the limit of result rows to return.
+  *@param afterOrderBy is true if this offset/limit comes after an ORDER BY.
   *@return the proper clause, with no padding spaces on either side.
   */
-  public String constructOffsetLimitClause(int offset, int limit)
+  public String constructOffsetLimitClause(int offset, int limit, boolean afterOrderBy)
   {
     StringBuilder sb = new StringBuilder();
     if (offset != 0)

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java?rev=1210307&r1=1210306&r2=1210307&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
Mon Dec  5 01:57:43 2011
@@ -1012,9 +1012,10 @@ public class DBInterfaceHSQLDB extends D
   * This method constructs an offset/limit clause in the proper manner for the database in
question.
   *@param offset is the starting offset number.
   *@param limit is the limit of result rows to return.
+  *@param afterOrderBy is true if this offset/limit comes after an ORDER BY.
   *@return the proper clause, with no padding spaces on either side.
   */
-  public String constructOffsetLimitClause(int offset, int limit)
+  public String constructOffsetLimitClause(int offset, int limit, boolean afterOrderBy)
   {
     StringBuilder sb = new StringBuilder();
     if (offset != 0)
@@ -1024,6 +1025,9 @@ public class DBInterfaceHSQLDB extends D
       if (offset != 0)
         sb.append(" ");
       sb.append("LIMIT ").append(Integer.toString(limit));
+      if (afterOrderBy)
+        // Hint to HSQLDB to use the order-by index
+        sb.append(" USING INDEX");
     }
     return sb.toString();
   }

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java?rev=1210307&r1=1210306&r2=1210307&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
Mon Dec  5 01:57:43 2011
@@ -572,9 +572,10 @@ public class DBInterfaceMySQL extends Da
   * This method constructs an offset/limit clause in the proper manner for the database in
question.
   *@param offset is the starting offset number.
   *@param limit is the limit of result rows to return.
+  *@param afterOrderBy is true if this offset/limit comes after an ORDER BY.
   *@return the proper clause, with no padding spaces on either side.
   */
-  public String constructOffsetLimitClause(int offset, int limit)
+  public String constructOffsetLimitClause(int offset, int limit, boolean afterOrderBy)
   {
     StringBuilder sb = new StringBuilder();
     if (offset != 0)

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java?rev=1210307&r1=1210306&r2=1210307&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
Mon Dec  5 01:57:43 2011
@@ -906,9 +906,10 @@ public class DBInterfacePostgreSQL exten
   * This method constructs an offset/limit clause in the proper manner for the database in
question.
   *@param offset is the starting offset number.
   *@param limit is the limit of result rows to return.
+  *@param afterOrderBy is true if this offset/limit comes after an ORDER BY.
   *@return the proper clause, with no padding spaces on either side.
   */
-  public String constructOffsetLimitClause(int offset, int limit)
+  public String constructOffsetLimitClause(int offset, int limit, boolean afterOrderBy)
   {
     StringBuilder sb = new StringBuilder();
     if (offset != 0)

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java?rev=1210307&r1=1210306&r2=1210307&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
Mon Dec  5 01:57:43 2011
@@ -474,6 +474,26 @@ public abstract class Database
     }
   }
 
+  /** Construct an offset/limit clause.
+  * This method constructs an offset/limit clause in the proper manner for the database in
question.
+  *@param offset is the starting offset number.
+  *@param limit is the limit of result rows to return.
+  *@return the proper clause, with no padding spaces on either side.
+  */
+  public String constructOffsetLimitClause(int offset, int limit)
+  {
+    return constructOffsetLimitClause(offset,limit,false);
+  }
+
+  /** Construct an offset/limit clause.
+  * This method constructs an offset/limit clause in the proper manner for the database in
question.
+  *@param offset is the starting offset number.
+  *@param limit is the limit of result rows to return.
+  *@param afterOrderBy is true if this offset/limit comes after an ORDER BY.
+  *@return the proper clause, with no padding spaces on either side.
+  */
+  public abstract String constructOffsetLimitClause(int offset, int limit, boolean afterOrderBy);
+
   /* Calculate the number of values a particular clause can have, given the values for all
the other clauses.
   * For example, if in the expression x AND y AND z, x has 2 values and z has 1, find out
how many values x can legally have
   * when using the buildConjunctionClause() method below.

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java?rev=1210307&r1=1210306&r2=1210307&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
Mon Dec  5 01:57:43 2011
@@ -281,10 +281,19 @@ public interface IDBInterface
   * This method constructs an offset/limit clause in the proper manner for the database in
question.
   *@param offset is the starting offset number.
   *@param limit is the limit of result rows to return.
+  *@param afterOrderBy is true if this offset/limit comes after an ORDER BY.
   *@return the proper clause, with no padding spaces on either side.
   */
-  public String constructOffsetLimitClause(int offset, int limit);
+  public String constructOffsetLimitClause(int offset, int limit, boolean afterOrderBy);
   
+  /** Construct an offset/limit clause.
+  * This method constructs an offset/limit clause in the proper manner for the database in
question.
+  *@param offset is the starting offset number.
+  *@param limit is the limit of result rows to return.
+  *@return the proper clause, with no padding spaces on either side.
+  */
+  public String constructOffsetLimitClause(int offset, int limit);
+
   /** Construct a 'distinct on (x)' filter.
   * This filter wraps a query and returns a new query whose results are similar to POSTGRESQL's
DISTINCT-ON feature.
   * Specifically, for each combination of the specified distinct fields in the result, only
the first such row is included in the final

Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1210307&r1=1210306&r2=1210307&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
(original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Mon Dec  5 01:57:43 2011
@@ -2126,7 +2126,7 @@ public class JobManager implements IJobM
       .append(jobQueue.statusField).append(" ASC,")
       .append(jobQueue.checkActionField).append(" ASC,")
       .append(jobQueue.checkTimeField).append(" ASC ")
-      .append(database.constructOffsetLimitClause(0,1));
+      .append(database.constructOffsetLimitClause(0,1,true));
 
     IResultSet set = database.performQuery(sb.toString(),list,null,null,1,null);
     if (set.getRowCount() > 0)
@@ -2242,7 +2242,7 @@ public class JobManager implements IJobM
 
       // Now we can tack the limit onto the query.  Before this point, remainingDocuments
would be crap
       int limitValue = vList.getRemainingDocuments();
-      sb.append(database.constructOffsetLimitClause(0,limitValue));
+      sb.append(database.constructOffsetLimitClause(0,limitValue,true));
 
       if (Logging.perf.isDebugEnabled())
       {

Modified: incubator/lcf/trunk/lib/hsqldb.jar
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/lib/hsqldb.jar?rev=1210307&r1=1210306&r2=1210307&view=diff
==============================================================================
Binary files - no diff available.

Modified: incubator/lcf/trunk/mvn-bootstrap.bat
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/mvn-bootstrap.bat?rev=1210307&r1=1210306&r2=1210307&view=diff
==============================================================================
--- incubator/lcf/trunk/mvn-bootstrap.bat (original)
+++ incubator/lcf/trunk/mvn-bootstrap.bat Mon Dec  5 01:57:43 2011
@@ -17,7 +17,7 @@ rem limitations under the License.
 cmd /c ant download-dependencies
 
 cmd /c mvn install:install-file -Dfile=lib/jdbcpool-0.99.jar -DgroupId=com.bitmechanic -DartifactId=jdbcpool
-Dversion=0.99 -Dpackaging=jar
-cmd /c mvn install:install-file -Dfile=lib/hsqldb.jar -DgroupId=org.hsqldb -DartifactId=hsqldb
-Dversion=2.2.5.10-27-2011 -Dpackaging=jar
+cmd /c mvn install:install-file -Dfile=lib/hsqldb.jar -DgroupId=org.hsqldb -DartifactId=hsqldb
-Dversion=2.2.6.12-04-2011 -Dpackaging=jar
 
 cmd /c mvn install:install-file -Dfile=lib/commons-httpclient-mcf.jar -DgroupId=commons-httpclient
-DartifactId=commons-httpclient-mcf -Dversion=3.1  -Dpackaging=jar
 cmd /c mvn install:install-file -Dfile=lib/xercesImpl-mcf.jar -DgroupId=xerces -DartifactId=xercesImpl-mcf
-Dversion=2.9.1  -Dpackaging=jar

Modified: incubator/lcf/trunk/mvn-bootstrap.sh
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/mvn-bootstrap.sh?rev=1210307&r1=1210306&r2=1210307&view=diff
==============================================================================
--- incubator/lcf/trunk/mvn-bootstrap.sh (original)
+++ incubator/lcf/trunk/mvn-bootstrap.sh Mon Dec  5 01:57:43 2011
@@ -24,7 +24,7 @@
 ant download-dependencies
 
 mvn install:install-file -Dfile=lib/jdbcpool-0.99.jar -DgroupId=com.bitmechanic -DartifactId=jdbcpool
-Dversion=0.99 -Dpackaging=jar
-mvn install:install-file -Dfile=lib/hsqldb.jar -DgroupId=org.hsqldb -DartifactId=hsqldb -Dversion=2.2.5.10-27-2011
-Dpackaging=jar
+mvn install:install-file -Dfile=lib/hsqldb.jar -DgroupId=org.hsqldb -DartifactId=hsqldb -Dversion=2.2.6.12-04-2011
-Dpackaging=jar
 
 mvn install:install-file -Dfile=lib/commons-httpclient-mcf.jar -DgroupId=commons-httpclient
-DartifactId=commons-httpclient-mcf -Dversion=3.1  -Dpackaging=jar
 mvn install:install-file -Dfile=lib/xercesImpl-mcf.jar -DgroupId=xerces -DartifactId=xercesImpl-mcf
-Dversion=2.9.1  -Dpackaging=jar

Modified: incubator/lcf/trunk/pom.xml
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/pom.xml?rev=1210307&r1=1210306&r2=1210307&view=diff
==============================================================================
--- incubator/lcf/trunk/pom.xml (original)
+++ incubator/lcf/trunk/pom.xml Mon Dec  5 01:57:43 2011
@@ -40,7 +40,7 @@
     <!-- dependency versions -->
     <junit.version>4.8.2</junit.version>
     <postgresql.version>8.4-702.jdbc4</postgresql.version>
-    <hsqldb.version>2.2.5.10-27-2011</hsqldb.version>
+    <hsqldb.version>2.2.6.12-04-2011</hsqldb.version>
     <derby.version>10.7.1.1</derby.version>
     <jetty.version>6.1.26</jetty.version>
   </properties>

Modified: incubator/lcf/trunk/site/src/documentation/content/xdocs/how-to-build-and-deploy.xml
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/site/src/documentation/content/xdocs/how-to-build-and-deploy.xml?rev=1210307&r1=1210306&r2=1210307&view=diff
==============================================================================
--- incubator/lcf/trunk/site/src/documentation/content/xdocs/how-to-build-and-deploy.xml (original)
+++ incubator/lcf/trunk/site/src/documentation/content/xdocs/how-to-build-and-deploy.xml Mon
Dec  5 01:57:43 2011
@@ -236,7 +236,7 @@ mvn install:install-file -Dfile=lib/comm
 mvn install:install-file -Dfile=lib/xercesImpl-mcf.jar -DgroupId=xerces -DartifactId=xercesImpl-mcf
-Dversion=2.9.1 -Dpackaging=jar
 mvn install:install-file -Dfile=lib/chemistry-opencmis-server-inmemory-war.war -DgroupId=org.apache.chemistry.opencmis
-DartifactId=chemistry-opencmis-server-inmemory-war -Dversion=0.5.0 -Dpackaging=war
 mvn install:install-file -Dfile=connectors/jcifs/jcifs/jcifs.jar -DgroupId=org.samba.jcifs
-DartifactId=jcifs -Dversion=1.3.17 -Dpackaging=jar
-mvn install:install-file -Dfile=lib/hsqldb.jar -DgroupId=org.hsqldb -DartifactId=hsqldb -Dversion=2.2.5.6-9-2011
-Dpackaging=jar
+mvn install:install-file -Dfile=lib/hsqldb.jar -DgroupId=org.hsqldb -DartifactId=hsqldb -Dversion=2.2.6.12-04-2011
-Dpackaging=jar
           </source>
         </section>
         <section>



Mime
View raw message