incubator-connectors-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1195471 [1/5] - in /incubator/lcf/trunk: ./ framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/ framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/ framework/core/src/main/java/org/apach...
Date Mon, 31 Oct 2011 14:26:36 GMT
Author: kwright
Date: Mon Oct 31 14:26:33 2011
New Revision: 1195471

URL: http://svn.apache.org/viewvc?rev=1195471&view=rev
Log:
More work for CONNECTORS-284 - review all indexes, and update all queries to present criteria in index order.

Added:
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ClauseDescription.java   (with props)
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java   (with props)
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/MultiClause.java   (with props)
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/NullCheckClause.java   (with props)
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/UnitaryClause.java   (with props)
    incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BaseDerby.java   (with props)
    incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BaseHSQLDB.java   (with props)
    incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BigCrawlDerbyTest.java   (with props)
    incubator/lcf/trunk/loadtests/rss/src/test/java/org/apache/manifoldcf/rss_loadtests/BigCrawlHSQLDBTest.java   (with props)
Modified:
    incubator/lcf/trunk/build.xml
    incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
    incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
    incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/Base.java
    incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BaseHSQLDB.java
    incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BasePostgresql.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopDeleteDeps.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/PrereqEventManager.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/ScheduleManager.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryHistoryManager.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/ThrottleSpecManager.java
    incubator/lcf/trunk/tests/cmis/src/test/java/org/apache/manifoldcf/cmis_tests/Base.java

Modified: incubator/lcf/trunk/build.xml
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/build.xml?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/build.xml (original)
+++ incubator/lcf/trunk/build.xml Mon Oct 31 14:26:33 2011
@@ -2301,7 +2301,7 @@
 
     <target name="run-rss-loadtests-derby" depends="compile-rss-loadtests,calculate-rss-tests-condition" if="rss-tests.include">
         <mkdir dir="test-output/rss"/>
-        <junit fork="true" maxmemory="256m" dir="test-output/rss" outputtoformatters="true" showoutput="true" haltonfailure="true">
+        <junit fork="true" maxmemory="384m" dir="test-output/rss" outputtoformatters="true" showoutput="true" haltonfailure="true">
             <classpath>
                 <fileset dir="framework/lib">
                     <include name="*.jar"/>
@@ -2328,13 +2328,15 @@
             </classpath>
             <formatter type="brief" usefile="false"/>
 
+            <test name="org.apache.manifoldcf.rss_loadtests.BigCrawlDerbyTest" todir="test-output/rss"/>
+
             <!-- MHL -->
         </junit>
     </target>
 
     <target name="run-rss-loadtests-postgresql" depends="compile-rss-loadtests,calculate-rss-tests-condition" if="rss-tests.include">
         <mkdir dir="test-postgresql-output/rss"/>
-        <junit fork="true" maxmemory="256m" dir="test-postgresql-output/rss" outputtoformatters="true" showoutput="true" haltonfailure="true">
+        <junit fork="true" maxmemory="384m" dir="test-postgresql-output/rss" outputtoformatters="true" showoutput="true" haltonfailure="true">
             <classpath>
                 <fileset dir="framework/lib">
                     <include name="*.jar"/>
@@ -2369,7 +2371,7 @@
 
     <target name="run-rss-loadtests-HSQLDB" depends="compile-rss-loadtests,calculate-rss-tests-condition" if="rss-tests.include">
         <mkdir dir="test-HSQLDB-output/rss"/>
-        <junit fork="true" maxmemory="256m" dir="test-HSQLDB-output/rss" outputtoformatters="true" showoutput="true" haltonfailure="true">
+        <junit fork="true" maxmemory="384m" dir="test-HSQLDB-output/rss" outputtoformatters="true" showoutput="true" haltonfailure="true">
             <classpath>
                 <fileset dir="framework/lib">
                     <include name="*.jar"/>
@@ -2395,7 +2397,9 @@
                 <pathelement location="build/rss-loadtests/classes"/>
             </classpath>
             <formatter type="brief" usefile="false"/>
-            
+
+            <test name="org.apache.manifoldcf.rss_loadtests.BigCrawlHSQLDBTest" todir="test-HSQLDB-output/rss"/>
+
             <!-- MHL -->
         </junit>
     </target>

Modified: incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java (original)
+++ incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java Mon Oct 31 14:26:33 2011
@@ -126,8 +126,9 @@ public class IncrementalIngester extends
       }
 
       // Now, do indexes
-      IndexDescription keyIndex = new IndexDescription(true,new String[]{outputConnNameField,docKeyField});
-      IndexDescription uriHashIndex = new IndexDescription(false,new String[]{uriHashField});
+      IndexDescription keyIndex = new IndexDescription(true,new String[]{docKeyField,outputConnNameField});
+      IndexDescription uriHashIndex = new IndexDescription(false,new String[]{uriHashField,outputConnNameField});
+      IndexDescription outputConnIndex = new IndexDescription(false,new String[]{outputConnNameField});
 
       // Get rid of indexes that shouldn't be there
       Map indexes = getTableIndexes(null,null);
@@ -141,6 +142,8 @@ public class IncrementalIngester extends
           keyIndex = null;
         else if (uriHashIndex != null && id.equals(uriHashIndex))
           uriHashIndex = null;
+        else if (outputConnIndex != null && id.equals(outputConnIndex))
+          outputConnIndex = null;
         else if (indexName.indexOf("_pkey") == -1)
           // This index shouldn't be here; drop it
           performRemoveIndex(indexName);
@@ -153,6 +156,9 @@ public class IncrementalIngester extends
       if (keyIndex != null)
         performAddIndex(null,keyIndex);
 
+      if (outputConnIndex != null)
+        performAddIndex(null,outputConnIndex);
+      
       // All done; break out of loop
       break;
     }
@@ -385,18 +391,20 @@ public class IncrementalIngester extends
     String oldURIHash = null;
     String oldOutputVersion = null;
 
-    // See what uri was used before for this doc, if any
-    ArrayList list = new ArrayList();
-    list.add(docKey);
-    list.add(connection.getName());
     
     while (true)
     {
       long sleepAmt = 0L;
       try
       {
+        // See what uri was used before for this doc, if any
+        ArrayList list = new ArrayList();
+        String query = buildConjunctionClause(list,new ClauseDescription[]{
+          new UnitaryClause(docKeyField,docKey),
+          new UnitaryClause(outputConnNameField,connection.getName())});
+          
         IResultSet set = performQuery("SELECT "+docURIField+","+uriHashField+","+lastOutputVersionField+" FROM "+getTableName()+
-          " WHERE "+docKeyField+"=? AND "+outputConnNameField+"=?",list,null,null);
+          " WHERE "+query,list,null,null);
 
         if (set.getRowCount() > 0)
         {
@@ -447,16 +455,17 @@ public class IncrementalIngester extends
     try
     {
 
+      ArrayList list = new ArrayList();
+      
       if (oldURI != null && (documentURI == null || !oldURI.equals(documentURI)))
       {
         // Delete all records from the database that match the old URI, except for THIS record.
         list.clear();
-        list.add(oldURIHash);
-	// Ideally, we should include the actual string, but that's hard because of database limitations
-        //list.add(oldURI);
+        String query = buildConjunctionClause(list,new ClauseDescription[]{
+          new UnitaryClause(uriHashField,"=",oldURIHash),
+          new UnitaryClause(outputConnNameField,"=",connection.getName())});
         list.add(docKey);
-        list.add(connection.getName());
-        performDelete("WHERE "+uriHashField+"=? AND "+ /* docURIField+"=? AND "+ */ docKeyField+"!=? AND "+outputConnNameField+"=?",list,null);
+        performDelete("WHERE "+query+" AND "+docKeyField+"!=?",list,null);
         removeDocument(connection,oldURI,oldOutputVersion,activities);
       }
 
@@ -464,12 +473,11 @@ public class IncrementalIngester extends
       {
         // Get rid of all records that match the NEW uri, except for this record.
         list.clear();
-        list.add(documentURIHash);
-        // Database limitations prevent this
-        // list.add(documentURI);
+        String query = buildConjunctionClause(list,new ClauseDescription[]{
+          new UnitaryClause(uriHashField,"=",documentURIHash),
+          new UnitaryClause(outputConnNameField,"=",connection.getName())});
         list.add(docKey);
-        list.add(connection.getName());
-        performDelete("WHERE "+uriHashField+"=? AND "+ /* docURIField+"=? AND "+ */ docKeyField+"!=? AND "+outputConnNameField+"=?",list,null);
+        performDelete("WHERE "+query+" AND "+ docKeyField+"!=?",list,null);
       }
 
       // Now, we know we are ready for the ingest.
@@ -524,8 +532,8 @@ public class IncrementalIngester extends
     beginTransaction();
     try
     {
-      int maxInClause = getMaxInClause();
-
+      int maxClauses;
+      
       HashMap docIDValues = new HashMap();
       int j = 0;
       while (j < identifierHashes.length)
@@ -541,51 +549,41 @@ public class IncrementalIngester extends
       Iterator iter = docIDValues.keySet().iterator();
       j = 0;
       ArrayList list = new ArrayList();
-      StringBuilder sb = new StringBuilder();
+      maxClauses = maxClausesRowIdsForDocIds(outputConnectionName);
       while (iter.hasNext())
       {
-        if (j == maxInClause)
+        if (j == maxClauses)
         {
-          findRowIdsForDocIds(outputConnectionName,rowIDSet,list,sb.toString());
+          findRowIdsForDocIds(outputConnectionName,rowIDSet,list);
           list.clear();
-          sb.setLength(0);
           j = 0;
         }
-        String idValue = (String)iter.next();
-        if (j > 0)
-          sb.append(',');
-        sb.append('?');
-        list.add(idValue);
+        list.add(iter.next());
         j++;
       }
 
       if (j > 0)
-        findRowIdsForDocIds(outputConnectionName,rowIDSet,list,sb.toString());
+        findRowIdsForDocIds(outputConnectionName,rowIDSet,list);
 
       // Now, break row id's into chunks too; submit one chunk at a time
       j = 0;
       list.clear();
-      sb.setLength(0);
       iter = rowIDSet.keySet().iterator();
+      maxClauses = maxClausesUpdateRowIds();
       while (iter.hasNext())
       {
-        if (j == maxInClause)
+        if (j == maxClauses)
         {
-          updateRowIds(list,sb.toString(),checkTime);
+          updateRowIds(list,checkTime);
           list.clear();
-          sb.setLength(0);
           j = 0;
         }
-        Long idValue = (Long)iter.next();
-        if (j > 0)
-          sb.append(',');
-        sb.append('?');
-        list.add(idValue);
+        list.add(iter.next());
         j++;
       }
 
       if (j > 0)
-        updateRowIds(list,sb.toString(),checkTime);
+        updateRowIds(list,checkTime);
     }
     catch (ManifoldCFException e)
     {
@@ -618,14 +616,25 @@ public class IncrementalIngester extends
     documentCheckMultiple(outputConnectionName,new String[]{identifierClass},new String[]{identifierHash},checkTime);
   }
 
+  /** Calculate the number of clauses.
+  */
+  protected int maxClausesUpdateRowIds()
+  {
+    return findConjunctionClauseMax(new ClauseDescription[]{});
+  }
+  
   /** Update a chunk of row ids.
   */
-  protected void updateRowIds(ArrayList list, String queryPart, long checkTime)
+  protected void updateRowIds(ArrayList list, long checkTime)
     throws ManifoldCFException
   {
+    ArrayList newList = new ArrayList();
+    String query = buildConjunctionClause(newList,new ClauseDescription[]{
+      new MultiClause(idField,list)});
+      
     HashMap map = new HashMap();
     map.put(lastIngestField,new Long(checkTime));
-    performUpdate(map,"WHERE "+idField+" IN ("+queryPart+")",list,null);
+    performUpdate(map,"WHERE "+query,newList,null);
   }
 
   /** Delete multiple documents from the search engine index.
@@ -755,8 +764,8 @@ public class IncrementalIngester extends
         // so the first step is to come up with ALL the doc hash values before looping
         // over them.
 
-        int maxInClause = getMaxInClause();
-
+        int maxClauses;
+        
         // Find all the documents that match this set of URIs
         HashMap docURIHashValues = new HashMap();
         HashMap docURIValues = new HashMap();
@@ -775,51 +784,41 @@ public class IncrementalIngester extends
         Iterator iter = docURIHashValues.keySet().iterator();
         j = 0;
         ArrayList hashList = new ArrayList();
-        StringBuilder sb = new StringBuilder();
+        maxClauses = maxClausesRowIdsForURIs(outputConnectionName);
         while (iter.hasNext())
         {
-          if (j == maxInClause)
+          if (j == maxClauses)
           {
-            findRowIdsForURIs(outputConnectionName,rowIDSet,docURIValues,hashList,sb.toString());
+            findRowIdsForURIs(outputConnectionName,rowIDSet,docURIValues,hashList);
             hashList.clear();
-            sb.setLength(0);
             j = 0;
           }
-          String hashValue = (String)iter.next();
-          if (j > 0)
-            sb.append(',');
-          sb.append('?');
-          hashList.add(hashValue);
+          hashList.add(iter.next());
           j++;
         }
 
         if (j > 0)
-          findRowIdsForURIs(outputConnectionName,rowIDSet,docURIValues,hashList,sb.toString());
+          findRowIdsForURIs(outputConnectionName,rowIDSet,docURIValues,hashList);
 
         // Next, go through the list of row IDs, and delete them in chunks
         j = 0;
         ArrayList list = new ArrayList();
-        sb.setLength(0);
         iter = rowIDSet.keySet().iterator();
+        maxClauses = maxClausesDeleteRowIds();
         while (iter.hasNext())
         {
-          if (j == maxInClause)
+          if (j == maxClauses)
           {
-            deleteRowIds(list,sb.toString());
+            deleteRowIds(list);
             list.clear();
-            sb.setLength(0);
             j = 0;
           }
-          Long idValue = (Long)iter.next();
-          if (j > 0)
-            sb.append(',');
-          sb.append('?');
-          list.add(idValue);
+          list.add(iter.next());
           j++;
         }
 
         if (j > 0)
-          deleteRowIds(list,sb.toString());
+          deleteRowIds(list);
 
         // Now, find the set of documents that remain that match the document identifiers.
         HashMap docIdValues = new HashMap();
@@ -837,51 +836,41 @@ public class IncrementalIngester extends
         iter = docIdValues.keySet().iterator();
         j = 0;
         list.clear();
-        sb.setLength(0);
+        maxClauses = maxClausesRowIdsForDocIds(outputConnectionName);
         while (iter.hasNext())
         {
-          if (j == maxInClause)
+          if (j == maxClauses)
           {
-            findRowIdsForDocIds(outputConnectionName,rowIDSet,list,sb.toString());
+            findRowIdsForDocIds(outputConnectionName,rowIDSet,list);
             list.clear();
-            sb.setLength(0);
             j = 0;
           }
-          String hashValue = (String)iter.next();
-          if (j > 0)
-            sb.append(',');
-          sb.append('?');
-          list.add(hashValue);
+          list.add(iter.next());
           j++;
         }
 
         if (j > 0)
-          findRowIdsForDocIds(outputConnectionName,rowIDSet,list,sb.toString());
+          findRowIdsForDocIds(outputConnectionName,rowIDSet,list);
 
         // Next, go through the list of row IDs, and delete them in chunks
         j = 0;
         list.clear();
-        sb.setLength(0);
         iter = rowIDSet.keySet().iterator();
+        maxClauses = maxClausesDeleteRowIds();
         while (iter.hasNext())
         {
-          if (j == maxInClause)
+          if (j == maxClauses)
           {
-            deleteRowIds(list,sb.toString());
+            deleteRowIds(list);
             list.clear();
-            sb.setLength(0);
             j = 0;
           }
-          Long idValue = (Long)iter.next();
-          if (j > 0)
-            sb.append(',');
-          sb.append('?');
-          list.add(idValue);
+          list.add(iter.next());
           j++;
         }
 
         if (j > 0)
-          deleteRowIds(list,sb.toString());
+          deleteRowIds(list);
 
       }
       catch (ManifoldCFException e)
@@ -905,16 +894,28 @@ public class IncrementalIngester extends
     }
   }
 
+  /** Calculate the clauses.
+  */
+  protected int maxClausesRowIdsForURIs(String outputConnectionName)
+  {
+    return findConjunctionClauseMax(new ClauseDescription[]{
+      new UnitaryClause(outputConnNameField,outputConnectionName)});
+  }
+  
   /** Given values and parameters corresponding to a set of hash values, add corresponding
   * table row id's to the output map.
   */
-  protected void findRowIdsForURIs(String outputConnectionName, HashMap rowIDSet, HashMap uris, ArrayList hashParamValues,
-    String paramList)
+  protected void findRowIdsForURIs(String outputConnectionName, HashMap rowIDSet, HashMap uris, ArrayList hashParamValues)
     throws ManifoldCFException
   {
-    hashParamValues.add(outputConnectionName);
+    ArrayList list = new ArrayList();
+    String query = buildConjunctionClause(list,new ClauseDescription[]{
+      new MultiClause(uriHashField,hashParamValues),
+      new UnitaryClause(outputConnNameField,outputConnectionName)});
+      
     IResultSet set = performQuery("SELECT "+idField+","+docURIField+" FROM "+
-      getTableName()+" WHERE "+uriHashField+" IN ("+paramList+") AND "+outputConnNameField+"=?",hashParamValues,null,null);
+      getTableName()+" WHERE "+query,list,null,null);
+      
     int i = 0;
     while (i < set.getRowCount())
     {
@@ -931,16 +932,28 @@ public class IncrementalIngester extends
     }
   }
 
+  /** Calculate the maximum number of doc ids we should use.
+  */
+  protected int maxClausesRowIdsForDocIds(String outputConnectionName)
+  {
+    return findConjunctionClauseMax(new ClauseDescription[]{
+      new UnitaryClause(outputConnNameField,outputConnectionName)});
+  }
+  
   /** Given values and parameters corresponding to a set of hash values, add corresponding
   * table row id's to the output map.
   */
-  protected void findRowIdsForDocIds(String outputConnectionName, HashMap rowIDSet, ArrayList paramValues,
-    String paramList)
+  protected void findRowIdsForDocIds(String outputConnectionName, HashMap rowIDSet, ArrayList paramValues)
     throws ManifoldCFException
   {
-    paramValues.add(outputConnectionName);
+    ArrayList list = new ArrayList();
+    String query = buildConjunctionClause(list,new ClauseDescription[]{
+      new MultiClause(docKeyField,paramValues),
+      new UnitaryClause(outputConnNameField,outputConnectionName)});
+      
     IResultSet set = performQuery("SELECT "+idField+" FROM "+
-      getTableName()+" WHERE "+docKeyField+" IN ("+paramList+") AND "+outputConnNameField+"=?",paramValues,null,null);
+      getTableName()+" WHERE "+query,list,null,null);
+      
     int i = 0;
     while (i < set.getRowCount())
     {
@@ -950,12 +963,22 @@ public class IncrementalIngester extends
     }
   }
 
+  /** Calculate the maximum number of clauses.
+  */
+  protected int maxClausesDeleteRowIds()
+  {
+    return findConjunctionClauseMax(new ClauseDescription[]{});
+  }
+    
   /** Delete a chunk of row ids.
   */
-  protected void deleteRowIds(ArrayList list, String queryPart)
+  protected void deleteRowIds(ArrayList list)
     throws ManifoldCFException
   {
-    performDelete("WHERE "+idField+" IN ("+queryPart+")",list,null);
+    ArrayList newList = new ArrayList();
+    String query = buildConjunctionClause(newList,new ClauseDescription[]{
+      new MultiClause(idField,list)});
+    performDelete("WHERE "+query,newList,null);
   }
 
   /** Delete a document from the search engine index.
@@ -993,28 +1016,23 @@ public class IncrementalIngester extends
     beginTransaction();
     try
     {
-      StringBuilder sb = new StringBuilder();
       ArrayList list = new ArrayList();
-      int maxCount = getMaxInClause();
+      int maxCount = maxClauseDocumentURIChunk(outputConnectionName);
       int j = 0;
       Iterator iter = map.keySet().iterator();
       while (iter.hasNext())
       {
         if (j == maxCount)
         {
-          getDocumentURIChunk(rval,map,outputConnectionName,sb.toString(),list);
+          getDocumentURIChunk(rval,map,outputConnectionName,list);
           j = 0;
-          sb.setLength(0);
           list.clear();
         }
-        if (j > 0)
-          sb.append(',');
-        sb.append('?');
         list.add(iter.next());
         j++;
       }
       if (j > 0)
-        getDocumentURIChunk(rval,map,outputConnectionName,sb.toString(),list);
+        getDocumentURIChunk(rval,map,outputConnectionName,list);
       return rval;
     }
     catch (ManifoldCFException e)
@@ -1116,28 +1134,23 @@ public class IncrementalIngester extends
     beginTransaction();
     try
     {
-      StringBuilder sb = new StringBuilder();
       ArrayList list = new ArrayList();
-      int maxCount = getMaxInClause();
+      int maxCount = maxClauseDocumentIngestDataChunk(outputConnectionName);
       int j = 0;
       Iterator iter = indexMap.keySet().iterator();
       while (iter.hasNext())
       {
         if (j == maxCount)
         {
-          getDocumentIngestDataChunk(rval,indexMap,outputConnectionName,sb.toString(),list);
+          getDocumentIngestDataChunk(rval,indexMap,outputConnectionName,list);
           j = 0;
-          sb.setLength(0);
           list.clear();
         }
-        if (j > 0)
-          sb.append(',');
-        sb.append('?');
         list.add(iter.next());
         j++;
       }
       if (j > 0)
-        getDocumentIngestDataChunk(rval,indexMap,outputConnectionName,sb.toString(),list);
+        getDocumentIngestDataChunk(rval,indexMap,outputConnectionName,list);
       return rval;
     }
     catch (ManifoldCFException e)
@@ -1212,51 +1225,56 @@ public class IncrementalIngester extends
     }
 
     // Get the chunk size
-    int maxInClause = getMaxInClause();
-
+    int maxClause = maxClauseGetIntervals(outputConnectionName);
 
     // Loop through the hash codes
     Iterator iter = idCodes.keySet().iterator();
     ArrayList list = new ArrayList();
-    StringBuilder sb = new StringBuilder();
     j = 0;
     while (iter.hasNext())
     {
-      if (j == maxInClause)
+      if (j == maxClause)
       {
-        getIntervals(rval,outputConnectionName,list,sb.toString(),returnMap);
+        getIntervals(rval,outputConnectionName,list,returnMap);
         list.clear();
-        sb.setLength(0);
         j = 0;
       }
 
-      if (j > 0)
-        sb.append(',');
-      sb.append('?');
       list.add(iter.next());
       j++;
     }
 
     if (j > 0)
-      getIntervals(rval,outputConnectionName,list,sb.toString(),returnMap);
+      getIntervals(rval,outputConnectionName,list,returnMap);
 
     return rval;
   }
 
-
+  /** Calculate the number of clauses.
+  */
+  protected int maxClauseGetIntervals(String outputConnectionName)
+  {
+    return findConjunctionClauseMax(new ClauseDescription[]{
+      new UnitaryClause(outputConnNameField,outputConnectionName)});
+    }
+    
   /** Query for and calculate the interval for a bunch of hashcodes.
   *@param rval is the array to stuff calculated return values into.
   *@param list is the list of parameters.
   *@param queryPart is the part of the query pertaining to the list of hashcodes
   *@param returnMap is a mapping from document id to rval index.
   */
-  protected void getIntervals(long[] rval, String outputConnectionName, ArrayList list, String queryPart, HashMap returnMap)
+  protected void getIntervals(long[] rval, String outputConnectionName, ArrayList list, HashMap returnMap)
     throws ManifoldCFException
   {
-    list.add(outputConnectionName);
+    ArrayList newList = new ArrayList();
+    String query = buildConjunctionClause(newList,new ClauseDescription[]{
+      new MultiClause(docKeyField,list),
+      new UnitaryClause(outputConnNameField,outputConnectionName)});
+      
     IResultSet set = performQuery("SELECT "+docKeyField+","+changeCountField+","+firstIngestField+","+lastIngestField+
-      " FROM "+getTableName()+" WHERE "+
-      docKeyField+" IN("+queryPart+") AND "+outputConnNameField+"=?",list,null,null);
+      " FROM "+getTableName()+" WHERE "+query,newList,null,null);
+
     int i = 0;
     while (i < set.getRowCount())
     {
@@ -1286,8 +1304,10 @@ public class IncrementalIngester extends
     HashMap map = new HashMap();
     map.put(lastVersionField,null);
     ArrayList list = new ArrayList();
-    list.add(outputConnectionName);
-    performUpdate(map,"WHERE "+outputConnNameField+"=?",list,null);
+    String query = buildConjunctionClause(list,new ClauseDescription[]{
+      new UnitaryClause(outputConnNameField,outputConnectionName)});
+      
+    performUpdate(map,"WHERE "+query,list,null);
   }
 
   /** Note the ingestion of a document, or the "update" of a document.
@@ -1353,10 +1373,11 @@ public class IncrementalIngester extends
         {
           // Look for existing row.
           ArrayList list = new ArrayList();
-          list.add(docKey);
-          list.add(outputConnectionName);
+          String query = buildConjunctionClause(list,new ClauseDescription[]{
+            new UnitaryClause(docKeyField,docKey),
+            new UnitaryClause(outputConnNameField,outputConnectionName)});
           IResultSet set = performQuery("SELECT "+idField+","+changeCountField+" FROM "+getTableName()+" WHERE "+
-            docKeyField+"=? AND "+outputConnNameField+"=? FOR UPDATE",list,null,null);
+            query+" FOR UPDATE",list,null,null);
           IResultRow row = null;
           if (set.getRowCount() > 0)
             row = set.getRow(0);
@@ -1365,11 +1386,12 @@ public class IncrementalIngester extends
           {
             // Update the record
             list.clear();
-            list.add(row.getValue(idField));
+            query = buildConjunctionClause(list,new ClauseDescription[]{
+              new UnitaryClause(idField,row.getValue(idField))});
             long changeCount = ((Long)row.getValue(changeCountField)).longValue();
             changeCount++;
             map.put(changeCountField,new Long(changeCount));
-            performUpdate(map,"WHERE "+idField+"=?",list,null);
+            performUpdate(map,"WHERE "+query,list,null);
             // Update successful!
             return;
           }
@@ -1452,18 +1474,31 @@ public class IncrementalIngester extends
     }
   }
 
+  /** Calculate how many clauses at a time
+  */
+  protected int maxClauseDocumentURIChunk(String outputConnectionName)
+  {
+    return findConjunctionClauseMax(new ClauseDescription[]{
+      new UnitaryClause(outputConnNameField,outputConnectionName)});
+  }
+  
   /** Get a chunk of document uris.
   *@param rval is the string array where the uris should be put.
   *@param map is the map from id to index.
   *@param clause is the in clause for the query.
   *@param list is the parameter list for the query.
   */
-  protected void getDocumentURIChunk(DeleteInfo[] rval, Map map, String outputConnectionName, String clause, ArrayList list)
+  protected void getDocumentURIChunk(DeleteInfo[] rval, Map map, String outputConnectionName, ArrayList list)
     throws ManifoldCFException
   {
-    list.add(outputConnectionName);
+    ArrayList newList = new ArrayList();
+    String query = buildConjunctionClause(newList,new ClauseDescription[]{
+      new MultiClause(docKeyField,list),
+      new UnitaryClause(outputConnNameField,outputConnectionName)});
+      
     IResultSet set = performQuery("SELECT "+docKeyField+","+docURIField+","+lastOutputVersionField+" FROM "+getTableName()+" WHERE "+
-      docKeyField+" IN ("+clause+") AND "+outputConnNameField+"=?",list,null,null);
+      query,newList,null,null);
+
     // Go through list and put into buckets.
     int i = 0;
     while (i < set.getRowCount())
@@ -1482,19 +1517,31 @@ public class IncrementalIngester extends
     }
   }
 
+  /** Count the clauses
+  */
+  protected int maxClauseDocumentIngestDataChunk(String outputConnectionName)
+  {
+    return findConjunctionClauseMax(new ClauseDescription[]{
+      new UnitaryClause(outputConnNameField,outputConnectionName)});
+  }
+  
   /** Get a chunk of document ingest data records.
   *@param rval is the document ingest status array where the data should be put.
   *@param map is the map from id to index.
   *@param clause is the in clause for the query.
   *@param list is the parameter list for the query.
   */
-  protected void getDocumentIngestDataChunk(DocumentIngestStatus[] rval, Map map, String outputConnectionName, String clause, ArrayList list)
+  protected void getDocumentIngestDataChunk(DocumentIngestStatus[] rval, Map map, String outputConnectionName, ArrayList list)
     throws ManifoldCFException
   {
+    ArrayList newList = new ArrayList();
+    String query = buildConjunctionClause(newList,new ClauseDescription[]{
+      new MultiClause(docKeyField,list),
+      new UnitaryClause(outputConnNameField,outputConnectionName)});
+      
     // Get the primary records associated with this hash value
-    list.add(outputConnectionName);
     IResultSet set = performQuery("SELECT "+idField+","+docKeyField+","+lastVersionField+","+lastOutputVersionField+","+authorityNameField+" FROM "+getTableName()+" WHERE "+
-      docKeyField+" IN ("+clause+") AND "+outputConnNameField+"=?",list,null,null);
+      query,newList,null,null);
 
     // Now, go through the original request once more, this time building the result
     int i = 0;

Modified: incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java (original)
+++ incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java Mon Oct 31 14:26:33 2011
@@ -102,7 +102,27 @@ public class OutputConnectionManager ext
         // Upgrade code, if needed, goes here.
       }
 
-      // Index management code goes here.
+      // Index management
+      IndexDescription classIndex = new IndexDescription(false,new String[]{classNameField});
+      
+      // Get rid of indexes that shouldn't be there
+      Map indexes = getTableIndexes(null,null);
+      Iterator iter = indexes.keySet().iterator();
+      while (iter.hasNext())
+      {
+        String indexName = (String)iter.next();
+        IndexDescription id = (IndexDescription)indexes.get(indexName);
+
+        if (classIndex != null && id.equals(classIndex))
+          classIndex = null;
+        else if (indexName.indexOf("_pkey") == -1)
+          // This index shouldn't be here; drop it
+          performRemoveIndex(indexName);
+      }
+
+      // Add the ones we didn't find
+      if (classIndex != null)
+        performAddIndex(null,classIndex);
 
       break;
     }
@@ -261,6 +281,7 @@ public class OutputConnectionManager ext
     while (true)
     {
       // Catch deadlock condition
+      long sleepAmt = 0L;
       try
       {
         ICacheHandle ch = cacheManager.enterCache(null,cacheKeys,getTransactionID());
@@ -275,9 +296,10 @@ public class OutputConnectionManager ext
             boolean isNew = object.getIsNew();
             // See whether the instance exists
             ArrayList params = new ArrayList();
-            params.add(object.getName());
+            String query = buildConjunctionClause(params,new ClauseDescription[]{
+              new UnitaryClause(nameField,object.getName())});
             IResultSet set = performQuery("SELECT * FROM "+getTableName()+" WHERE "+
-              nameField+"=? FOR UPDATE",params,null,null);
+              query+" FOR UPDATE",params,null,null);
             HashMap values = new HashMap();
             values.put(descriptionField,object.getDescription());
             values.put(classNameField,object.getClassName());
@@ -300,8 +322,9 @@ public class OutputConnectionManager ext
 
               // Update
               params.clear();
-              params.add(object.getName());
-              performUpdate(values," WHERE "+nameField+"=?",params,null);
+              query = buildConjunctionClause(params,new ClauseDescription[]{
+                new UnitaryClause(nameField,object.getName())});
+              performUpdate(values," WHERE "+query,params,null);
             }
             else
             {
@@ -347,14 +370,11 @@ public class OutputConnectionManager ext
         // Is this a deadlock exception?  If so, we want to try again.
         if (e.getErrorCode() != ManifoldCFException.DATABASE_TRANSACTION_ABORT)
           throw e;
-        try
-        {
-          ManifoldCF.sleep((long)(random.nextDouble() * 60000.0 + 500.0));
-        }
-        catch (InterruptedException e2)
-        {
-          throw new ManifoldCFException(e2.getMessage(),e2,ManifoldCFException.INTERRUPTED);
-        }
+        sleepAmt = getSleepAmt();
+      }
+      finally
+      {
+        sleepFor(sleepAmt);
       }
     }
   }
@@ -381,8 +401,9 @@ public class OutputConnectionManager ext
           throw new ManifoldCFException("Can't delete output connection '"+name+"': existing entities refer to it");
         ManifoldCF.noteConfigurationChange();
         ArrayList params = new ArrayList();
-        params.add(name);
-        performDelete("WHERE "+nameField+"=?",params,null);
+        String query = buildConjunctionClause(params,new ClauseDescription[]{
+          new UnitaryClause(nameField,name)});
+        performDelete("WHERE "+query,params,null);
         cacheManager.invalidateKeys(ch);
       }
       catch (ManifoldCFException e)
@@ -418,8 +439,9 @@ public class OutputConnectionManager ext
     ssb.add(getOutputConnectionsKey());
     StringSet localCacheKeys = new StringSet(ssb);
     ArrayList params = new ArrayList();
-    params.add(className);
-    IResultSet set = performQuery("SELECT "+nameField+" FROM "+getTableName()+" WHERE "+classNameField+"=?",params,
+    String query = buildConjunctionClause(params,new ClauseDescription[]{
+      new UnitaryClause(classNameField,className)});
+    IResultSet set = performQuery("SELECT "+nameField+" FROM "+getTableName()+" WHERE "+query,params,
       localCacheKeys,null);
     String[] rval = new String[set.getRowCount()];
     int i = 0;
@@ -447,8 +469,9 @@ public class OutputConnectionManager ext
       ssb.add(getOutputConnectionKey(name));
       StringSet localCacheKeys = new StringSet(ssb);
       ArrayList params = new ArrayList();
-      params.add(name);
-      IResultSet set = performQuery("SELECT "+classNameField+" FROM "+getTableName()+" WHERE "+nameField+"=?",params,
+      String query = buildConjunctionClause(params,new ClauseDescription[]{
+        new UnitaryClause(nameField,name)});
+      IResultSet set = performQuery("SELECT "+classNameField+" FROM "+getTableName()+" WHERE "+query,params,
         localCacheKeys,null);
       if (set.getRowCount() == 0)
         throw new ManifoldCFException("No such connection: '"+name+"'");
@@ -524,28 +547,23 @@ public class OutputConnectionManager ext
     try
     {
       i = 0;
-      StringBuilder sb = new StringBuilder();
       ArrayList params = new ArrayList();
       int j = 0;
-      int maxIn = getMaxInClause();
+      int maxIn = maxClauseGetOutputConnectionsChunk();
       while (i < connectionNames.length)
       {
         if (j == maxIn)
         {
-          getOutputConnectionsChunk(rval,returnIndex,sb.toString(),params);
-          sb.setLength(0);
+          getOutputConnectionsChunk(rval,returnIndex,params);
           params.clear();
           j = 0;
         }
-        if (j > 0)
-          sb.append(',');
-        sb.append('?');
         params.add(connectionNames[i]);
         i++;
         j++;
       }
       if (j > 0)
-        getOutputConnectionsChunk(rval,returnIndex,sb.toString(),params);
+        getOutputConnectionsChunk(rval,returnIndex,params);
       return rval;
     }
     catch (Error e)
@@ -564,18 +582,26 @@ public class OutputConnectionManager ext
     }
   }
 
+  /** Calculate max number of clauses to send to getOutputConnectionsChunk.
+  */
+  protected int maxClauseGetOutputConnectionsChunk()
+  {
+    return findConjunctionClauseMax(new ClauseDescription[]{});
+  }
+    
   /** Read a chunk of output connections.
   *@param rval is the place to put the read policies.
   *@param returnIndex is a map from the object id (resource id) and the rval index.
-  *@param idList is the list of id's.
   *@param params is the set of parameters.
   */
-  protected void getOutputConnectionsChunk(OutputConnection[] rval, Map returnIndex, String idList, ArrayList params)
+  protected void getOutputConnectionsChunk(OutputConnection[] rval, Map returnIndex, ArrayList params)
     throws ManifoldCFException
   {
-    IResultSet set;
-    set = performQuery("SELECT * FROM "+getTableName()+" WHERE "+
-      nameField+" IN ("+idList+")",params,null,null);
+    ArrayList list = new ArrayList();
+    String query = buildConjunctionClause(list,new ClauseDescription[]{
+      new MultiClause(nameField,params)});
+    IResultSet set = performQuery("SELECT * FROM "+getTableName()+" WHERE "+
+      query,list,null,null);
     int i = 0;
     while (i < set.getRowCount())
     {

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java Mon Oct 31 14:26:33 2011
@@ -364,6 +364,25 @@ public class BaseTable
       orderFields,orderFieldsAscending,otherFields);
   }
 
+  /* Calculate the number of values a particular clause can have, given the values for all the other clauses.
+  * For example, if in the expression x AND y AND z, x has 2 values and z has 1, find out how many values x can legally have
+  * when using the buildConjunctionClause() method below.
+  */
+  public int findConjunctionClauseMax(ClauseDescription[] otherClauseDescriptions)
+  {
+    return dbInterface.findConjunctionClauseMax(otherClauseDescriptions);
+  }
+  
+  /* Construct a conjunction clause, e.g. x AND y AND z, where there is expected to be an index (x,y,z,...), and where x, y, or z
+  * can have multiple distinct values, The proper implementation of this method differs from database to database, because some databases
+  * only permit index operations when there are OR's between clauses, such as x1 AND y1 AND z1 OR x2 AND y2 AND z2 ..., where others
+  * only recognize index operations when there are lists specified for each, such as x IN (x1,x2) AND y IN (y1,y2) AND z IN (z1,z2).
+  */
+  public String buildConjunctionClause(List outputParameters, ClauseDescription[] clauseDescriptions)
+  {
+    return dbInterface.buildConjunctionClause(outputParameters,clauseDescriptions);
+  }
+
   /** Obtain the maximum number of individual items that should be
   * present in an IN clause.  Exceeding this amount will potentially cause the query performance
   * to drop.

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java Mon Oct 31 14:26:33 2011
@@ -108,6 +108,16 @@ public class DBInterfaceDerby extends Da
     this.password = password;
   }
 
+  public DBInterfaceDerby(IThreadContext tc, String databaseName)
+    throws ManifoldCFException
+  {
+    super(tc,_url+databaseName,_driver,databaseName,"","");
+    cacheKey = CacheKeyFactory.makeDatabaseKey(this.databaseName);
+    lockManager = LockManagerFactory.make(tc);
+    this.userName = "";
+    this.password = "";
+  }
+  
   /** Initialize.  This method is called once per JVM instance, in order to set up
   * database communication.
   */
@@ -704,7 +714,7 @@ public class DBInterfaceDerby extends Da
   public void createUserAndDatabase(String adminUserName, String adminPassword, StringSet invalidateKeys)
     throws ManifoldCFException
   {
-    Database rootDatabase = new Database(context,_url+databaseName,_driver,databaseName,"","");
+    Database rootDatabase = new DBInterfaceDerby(context,databaseName);
     IResultSet set = rootDatabase.executeQuery("VALUES SYSCS_UTIL.SYSCS_GET_DATABASE_PROPERTY('derby.user."+userName+"')",null,null,null,null,true,-1,null,null);
     if (set.getRowCount() == 0)
     {

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java Mon Oct 31 14:26:33 2011
@@ -1000,6 +1000,104 @@ public class DBInterfaceHSQLDB extends D
     return sb.toString();
   }
 
+  /* Calculate the number of values a particular clause can have, given the values for all the other clauses.
+  * For example, if in the expression x AND y AND z, x has 2 values and z has 1, find out how many values x can legally have
+  * when using the buildConjunctionClause() method below.
+  */
+  @Override
+  public int findConjunctionClauseMax(ClauseDescription[] otherClauseDescriptions)
+  {
+    // Special handling when there's only 1
+    if (otherClauseDescriptions.length == 0)
+      return super.findConjunctionClauseMax(otherClauseDescriptions);
+
+    // Since it's an OR clause we have to generate, figure out how many clauses are generated by the others,
+    // and work back from there.
+    int number = 1;
+    for (int i = 0 ; i < otherClauseDescriptions.length ; i++)
+    {
+      ClauseDescription otherClause = otherClauseDescriptions[i];
+      List values = otherClause.getValues();
+      if (values != null)
+        number *= values.size();
+    }
+    int rval = getMaxOrClause() / number;
+    if (rval == 0)
+      rval = 1;
+    return rval;
+  }
+  
+  /* Construct a conjunction clause, e.g. x AND y AND z, where there is expected to be an index (x,y,z,...), and where x, y, or z
+  * can have multiple distinct values, The proper implementation of this method differs from database to database, because some databases
+  * only permit index operations when there are OR's between clauses, such as x1 AND y1 AND z1 OR x2 AND y2 AND z2 ..., where others
+  * only recognize index operations when there are lists specified for each, such as x IN (x1,x2) AND y IN (y1,y2) AND z IN (z1,z2).
+  */
+  @Override
+  public String buildConjunctionClause(List outputParameters, ClauseDescription[] clauseDescriptions)
+  {
+    // Special handling when there's only 1
+    if (clauseDescriptions.length == 1)
+      return super.buildConjunctionClause(outputParameters,clauseDescriptions);
+    
+    StringBuilder sb = new StringBuilder("(");
+    int[] counters = new int[clauseDescriptions.length];
+    for (int i = 0 ; i < counters.length ; i++)
+    {
+      counters[i] = 0;
+    }
+    
+    boolean isFirst = true;
+    while (true)
+    {
+      // Add this clause in
+      if (isFirst)
+        isFirst = false;
+      else
+        sb.append(" OR ");
+      for (int i = 0 ; i < counters.length ; i++)
+      {
+        ClauseDescription cd = clauseDescriptions[i];
+        if (i > 0)
+          sb.append(" AND ");
+        List values = cd.getValues();
+        String joinColumn = cd.getJoinColumnName();
+        sb.append(cd.getColumnName()).append(cd.getOperation());
+        if (values != null)
+        {
+          sb.append("?");
+          outputParameters.add(values.get(counters[i]));
+        }
+        else if (joinColumn != null)
+          sb.append(joinColumn);
+      }
+    
+      // Now, increment the counters
+      int j = 0;
+      while (true)
+      {
+        if (j == counters.length)
+        {
+          sb.append(")");
+          return sb.toString();
+        }
+        counters[j]++;
+        ClauseDescription cd = clauseDescriptions[j];
+        List values = cd.getValues();
+        int size = 1;
+        if (values != null)
+          size = values.size();
+        if (counters[j] < size)
+          break;
+        j++;
+        for (int k = 0 ; k < j ; k++)
+        {
+          counters[k] = 0;
+        }
+        // Loop around to carry a one to the j'th counter
+      }
+    }
+  }
+
   /** Obtain the maximum number of individual items that should be
   * present in an IN clause.  Exceeding this amount will potentially cause the query performance
   * to drop.

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java Mon Oct 31 14:26:33 2011
@@ -387,7 +387,7 @@ public class DBInterfaceMySQL extends Da
     throws ManifoldCFException
   {
     // Connect to super database
-    Database masterDatabase = new Database(context,_url+"mysql",_driver,"mysql",adminUserName,adminPassword);
+    Database masterDatabase = new DBInterfaceMySQL(context,"mysql",adminUserName,adminPassword);
     List list = new ArrayList();
     list.add("utf8");
     masterDatabase.executeQuery("CREATE DATABASE "+databaseName+" CHARACTER SET ?",list,
@@ -412,7 +412,7 @@ public class DBInterfaceMySQL extends Da
     throws ManifoldCFException
   {
     // Connect to super database
-    Database masterDatabase = new Database(context,_url+"mysql",_driver,"mysql",adminUserName,adminPassword);
+    Database masterDatabase = new DBInterfaceMySQL(context,"mysql",adminUserName,adminPassword);
     masterDatabase.executeQuery("DROP DATABASE "+databaseName,null,null,invalidateKeys,null,false,0,null,null);
   }
 

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java Mon Oct 31 14:26:33 2011
@@ -523,7 +523,7 @@ public class DBInterfacePostgreSQL exten
     throws ManifoldCFException
   {
     // Create a connection to the master database, using the credentials supplied
-    Database masterDatabase = new Database(context,getJdbcUrl("template1"),_driver,"template1",adminUserName,adminPassword);
+    Database masterDatabase = new DBInterfacePostgreSQL(context,"template1",adminUserName,adminPassword);
     try
     {
       // Create user
@@ -577,7 +577,7 @@ public class DBInterfacePostgreSQL exten
     throws ManifoldCFException
   {
     // Create a connection to the master database, using the credentials supplied
-    Database masterDatabase = new Database(context,getJdbcUrl("template1"),_driver,"template1",adminUserName,adminPassword);
+    Database masterDatabase = new DBInterfacePostgreSQL(context,"template1",adminUserName,adminPassword);
     try
     {
       // Drop database

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java Mon Oct 31 14:26:33 2011
@@ -33,7 +33,7 @@ import javax.sql.*;
 * the transaction management will get screwed up (i.e. nobody will know what happened to the connection
 * handles...)
 */
-public class Database
+public abstract class Database
 {
   public static final String _rcsid = "@(#)$Id: Database.java 988245 2010-08-23 18:39:35Z kwright $";
 
@@ -50,6 +50,8 @@ public class Database
   protected int delayedTransactionDepth = 0;
   protected Map<String,Modifications> modificationsSet = new HashMap<String,Modifications>();
 
+  protected long maxQueryTime;
+  
   protected static Random random = new Random();
 
   protected final static String _TRANSACTION_ = "_TRANSACTION_";
@@ -63,6 +65,8 @@ public class Database
     this.databaseName = databaseName;
     this.userName = userName;
     this.password = password;
+    
+    this.maxQueryTime = ((long)ManifoldCF.getIntProperty(ManifoldCF.databaseQueryMaxTimeProperty,60)) * 1000L;
     this.cacheManager = CacheManagerFactory.make(context);
   }
 
@@ -426,6 +430,71 @@ public class Database
     }
   }
 
+  /* Calculate the number of values a particular clause can have, given the values for all the other clauses.
+  * For example, if in the expression x AND y AND z, x has 2 values and z has 1, find out how many values x can legally have
+  * when using the buildConjunctionClause() method below.
+  */
+  public int findConjunctionClauseMax(ClauseDescription[] otherClauseDescriptions)
+  {
+    // Base implementation uses "IN" for multiple values, since this seems to be widely accepted.
+    return getMaxInClause();
+  }
+  
+  /** Obtain the maximum number of individual items that should be
+  * present in an IN clause.  Exceeding this amount will potentially cause the query performance
+  * to drop.
+  *@return the maximum number of IN clause members.
+  */
+  public abstract int getMaxInClause();
+
+  /* Construct a conjunction clause, e.g. x AND y AND z, where there is expected to be an index (x,y,z,...), and where x, y, or z
+  * can have multiple distinct values, The proper implementation of this method differs from database to database, because some databases
+  * only permit index operations when there are OR's between clauses, such as x1 AND y1 AND z1 OR x2 AND y2 AND z2 ..., where others
+  * only recognize index operations when there are lists specified for each, such as x IN (x1,x2) AND y IN (y1,y2) AND z IN (z1,z2).
+  */
+  public String buildConjunctionClause(List outputParameters, ClauseDescription[] clauseDescriptions)
+  {
+    // Base implementation uses "IN" for multiple values, since this seems to be widely accepted.
+    StringBuilder sb = new StringBuilder();
+    for (int i = 0 ; i < clauseDescriptions.length ; i++)
+    {
+      ClauseDescription cd = clauseDescriptions[i];
+      if (i > 0)
+        sb.append(" AND ");
+      sb.append(cd.getColumnName());
+      String operation = cd.getOperation();
+      List values = cd.getValues();
+      String joinColumn = cd.getJoinColumnName();
+      if (values != null)
+      {
+        if (values.size() > 1)
+        {
+          sb.append(" IN (");
+          for (int j = 0 ; j < values.size() ; j++)
+          {
+            if (j > 0)
+              sb.append(",");
+            sb.append("?");
+            outputParameters.add(values.get(j));
+          }
+          sb.append(")");
+        }
+        else
+        {
+          sb.append(operation).append("?");
+          outputParameters.add(values.get(0));
+        }
+      }
+      else if (joinColumn != null)
+      {
+        sb.append(operation).append(joinColumn);
+      }
+      else
+        sb.append(operation);
+    }
+    return sb.toString();
+  }
+
   /** Class to keep track of modifications while we're in a transaction.
   */
   protected static class Modifications
@@ -1231,10 +1300,10 @@ public class Database
           description.getMaxReturn(),spec,limit);
 
         long endTime = System.currentTimeMillis();
-        if (endTime-startTime > 60000L && description.getQuery().length() >= 6 &&
+        if (endTime-startTime > database.maxQueryTime && description.getQuery().length() >= 6 &&
           ("SELECT".equalsIgnoreCase(description.getQuery().substring(0,6)) || "UPDATE".equalsIgnoreCase(description.getQuery().substring(0,6))))
         {
-          Logging.db.warn("Found a query that took more than a minute ("+new Long(endTime-startTime).toString()+" ms): ["+description.getQuery()+"]");
+          Logging.db.warn("Found a long-running query ("+new Long(endTime-startTime).toString()+" ms): ["+description.getQuery()+"]");
           if (description.getParameters() != null)
           {
             int j = 0;

Added: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ClauseDescription.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ClauseDescription.java?rev=1195471&view=auto
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ClauseDescription.java (added)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ClauseDescription.java Mon Oct 31 14:26:33 2011
@@ -0,0 +1,39 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.core.interfaces;
+
+import java.util.*;
+
+public interface ClauseDescription
+{
+  public static final String _rcsid = "@(#)$Id$";
+
+  /** Get the column name */
+  public String getColumnName();
+  
+  /** Get the operation string */
+  public String getOperation();
+  
+  /** Get the individual values */
+  public List getValues();
+  
+  /** Get the join column name */
+  public String getJoinColumnName();
+  
+}

Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ClauseDescription.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ClauseDescription.java
------------------------------------------------------------------------------
    svn:keywords = Id

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java Mon Oct 31 14:26:33 2011
@@ -302,6 +302,19 @@ public interface IDBInterface
   public String constructDistinctOnClause(List outputParameters, String baseQuery, List baseParameters,
     String[] distinctFields, String[] orderFields, boolean[] orderFieldsAscending, Map<String,String> otherFields);
   
+  /* Calculate the number of values a particular clause can have, given the values for all the other clauses.
+  * For example, if in the expression x AND y AND z, x has 2 values and z has 1, find out how many values x can legally have
+  * when using the buildConjunctionClause() method below.
+  */
+  public int findConjunctionClauseMax(ClauseDescription[] otherClauseDescriptions);
+  
+  /* Construct a conjunction clause, e.g. x AND y AND z, where there is expected to be an index (x,y,z,...), and where x, y, or z
+  * can have multiple distinct values, The proper implementation of this method differs from database to database, because some databases
+  * only permit index operations when there are OR's between clauses, such as x1 AND y1 AND z1 OR x2 AND y2 AND z2 ..., where others
+  * only recognize index operations when there are lists specified for each, such as x IN (x1,x2) AND y IN (y1,y2) AND z IN (z1,z2).
+  */
+  public String buildConjunctionClause(List outputParameters, ClauseDescription[] clauseDescriptions);
+  
   /** Obtain the maximum number of individual items that should be
   * present in an IN clause.  Exceeding this amount will potentially cause the query performance
   * to drop.

Added: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java?rev=1195471&view=auto
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java (added)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java Mon Oct 31 14:26:33 2011
@@ -0,0 +1,61 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.core.interfaces;
+
+import java.util.*;
+
+public class JoinClause implements ClauseDescription
+{
+  public static final String _rcsid = "@(#)$Id$";
+
+  protected String columnName;
+  protected String joinColumnName;
+  
+  /** Construct */
+  public JoinClause(String columnName, String joinColumnName)
+  {
+    this.columnName = columnName;
+    this.joinColumnName = joinColumnName;
+  }
+  
+  /** Get the column name */
+  public String getColumnName()
+  {
+    return columnName;
+  }
+  
+  /** Get the operation string */
+  public String getOperation()
+  {
+    return "=";
+  }
+  
+  /** Get the individual values */
+  public List getValues()
+  {
+    return null;
+  }
+  
+  /** Get the join column name */
+  public String getJoinColumnName()
+  {
+    return joinColumnName;
+  }
+
+}

Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java
------------------------------------------------------------------------------
    svn:keywords = Id

Added: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/MultiClause.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/MultiClause.java?rev=1195471&view=auto
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/MultiClause.java (added)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/MultiClause.java Mon Oct 31 14:26:33 2011
@@ -0,0 +1,70 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.core.interfaces;
+
+import java.util.*;
+
+public class MultiClause implements ClauseDescription
+{
+  public static final String _rcsid = "@(#)$Id$";
+
+  protected String columnName;
+  protected List values;
+  
+  /** Construct */
+  public MultiClause(String columnName, List values)
+  {
+    this.columnName = columnName;
+    this.values = values;
+  }
+  
+  public MultiClause(String columnName, Object[] values)
+  {
+    this.columnName = columnName;
+    this.values = new ArrayList();
+    for (int i = 0 ; i < values.length ; i++)
+    {
+      this.values.add(values[i]);
+    }
+  }
+  
+  /** Get the column name */
+  public String getColumnName()
+  {
+    return columnName;
+  }
+  
+  /** Get the operation string */
+  public String getOperation()
+  {
+    return "=";
+  }
+  
+  /** Get the individual values */
+  public List getValues()
+  {
+    return values;
+  }
+  
+  /** Get the join column name */
+  public String getJoinColumnName()
+  {
+    return null;
+  }
+}

Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/MultiClause.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/MultiClause.java
------------------------------------------------------------------------------
    svn:keywords = Id

Added: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/NullCheckClause.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/NullCheckClause.java?rev=1195471&view=auto
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/NullCheckClause.java (added)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/NullCheckClause.java Mon Oct 31 14:26:33 2011
@@ -0,0 +1,61 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.core.interfaces;
+
+import java.util.*;
+
+public class NullCheckClause implements ClauseDescription
+{
+  public static final String _rcsid = "@(#)$Id$";
+
+  protected String columnName;
+  protected boolean isNull;
+  
+  /** Construct */
+  public NullCheckClause(String columnName, boolean isNull)
+  {
+    this.columnName = columnName;
+    this.isNull = isNull;
+  }
+  
+  /** Get the column name */
+  public String getColumnName()
+  {
+    return columnName;
+  }
+  
+  /** Get the operation string */
+  public String getOperation()
+  {
+    return isNull?" IS NULL":" IS NOT NULL";
+  }
+  
+  /** Get the individual values */
+  public List getValues()
+  {
+    return null;
+  }
+  
+  /** Get the join column name */
+  public String getJoinColumnName()
+  {
+    return null;
+  }
+
+}

Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/NullCheckClause.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/NullCheckClause.java
------------------------------------------------------------------------------
    svn:keywords = Id

Added: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/UnitaryClause.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/UnitaryClause.java?rev=1195471&view=auto
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/UnitaryClause.java (added)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/UnitaryClause.java Mon Oct 31 14:26:33 2011
@@ -0,0 +1,68 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.core.interfaces;
+
+import java.util.*;
+
+public class UnitaryClause implements ClauseDescription
+{
+  public static final String _rcsid = "@(#)$Id$";
+
+  protected String columnName;
+  protected String operationName;
+  protected List values;
+  
+  /** Construct */
+  public UnitaryClause(String columnName, String operationName, Object value)
+  {
+    this.columnName = columnName;
+    this.operationName = operationName;
+    this.values = new ArrayList();
+    this.values.add(value);
+  }
+  
+  public UnitaryClause(String columnName, Object value)
+  {
+    this(columnName,"=",value);
+  }
+  
+  /** Get the column name */
+  public String getColumnName()
+  {
+    return columnName;
+  }
+  
+  /** Get the operation string */
+  public String getOperation()
+  {
+    return operationName;
+  }
+  
+  /** Get the individual values */
+  public List getValues()
+  {
+    return values;
+  }
+  
+  /** Get the join column name */
+  public String getJoinColumnName()
+  {
+    return null;
+  }
+}

Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/UnitaryClause.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/UnitaryClause.java
------------------------------------------------------------------------------
    svn:keywords = Id

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java Mon Oct 31 14:26:33 2011
@@ -109,6 +109,10 @@ public class ManifoldCF
   /** Database handle timeout property */
   public static final String databaseHandleTimeoutProperty = "org.apache.manifoldcf.database.handletimeout";
 
+  // Database performance monitoring properties
+  /** Elapsed time a query can take before a warning is output to the log, in seconds */
+  public static final String databaseQueryMaxTimeProperty = "org.apache.manifoldcf.database.maxquerytime";
+  
   // Log configuration properties
   /** Location of log configuration file */
   public static final String logConfigFileProperty = "org.apache.manifoldcf.logconfigfile";

Modified: incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/Base.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/Base.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/Base.java (original)
+++ incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/Base.java Mon Oct 31 14:26:33 2011
@@ -95,6 +95,12 @@ public class Base
       "<configuration>\n"+
       "  <property name=\"org.apache.manifoldcf.databaseimplementationclass\" value=\"org.apache.manifoldcf.core.database.DBInterfaceDerby\"/>\n" +
       "  <property name=\"org.apache.manifoldcf.derbydatabasepath\" value=\""+currentPathString.replaceAll("\\\\","/")+"\"/>\n" +
+      "  <property name=\"org.apache.manifoldcf.database.maxquerytime\" value=\"30\"/>\n" +
+      "  <property name=\"org.apache.manifoldcf.crawler.threads\" value=\"30\"/>\n" +
+      "  <property name=\"org.apache.manifoldcf.crawler.expirethreads\" value=\"10\"/>\n" +
+      "  <property name=\"org.apache.manifoldcf.crawler.cleanupthreads\" value=\"10\"/>\n" +
+      "  <property name=\"org.apache.manifoldcf.crawler.deletethreads\" value=\"10\"/>\n" +
+      "  <property name=\"org.apache.manifoldcf.database.maxhandles\" value=\"80\"/>\n" +
       "  <property name=\"org.apache.manifoldcf.logconfigfile\" value=\""+loggingFile.getAbsolutePath().replaceAll("\\\\","/")+"\"/>\n" +
       "</configuration>\n");
 

Modified: incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BaseHSQLDB.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BaseHSQLDB.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BaseHSQLDB.java (original)
+++ incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BaseHSQLDB.java Mon Oct 31 14:26:33 2011
@@ -95,6 +95,12 @@ public class BaseHSQLDB
       "<configuration>\n"+
       "  <property name=\"org.apache.manifoldcf.databaseimplementationclass\" value=\"org.apache.manifoldcf.core.database.DBInterfaceHSQLDB\"/>\n" +
       "  <property name=\"org.apache.manifoldcf.hsqldbdatabasepath\" value=\""+currentPathString.replaceAll("\\\\","/")+"\"/>\n" +
+      "  <property name=\"org.apache.manifoldcf.database.maxquerytime\" value=\"30\"/>\n" +
+      "  <property name=\"org.apache.manifoldcf.crawler.threads\" value=\"30\"/>\n" +
+      "  <property name=\"org.apache.manifoldcf.crawler.expirethreads\" value=\"10\"/>\n" +
+      "  <property name=\"org.apache.manifoldcf.crawler.cleanupthreads\" value=\"10\"/>\n" +
+      "  <property name=\"org.apache.manifoldcf.crawler.deletethreads\" value=\"10\"/>\n" +
+      "  <property name=\"org.apache.manifoldcf.database.maxhandles\" value=\"80\"/>\n" +
       "  <property name=\"org.apache.manifoldcf.logconfigfile\" value=\""+loggingFile.getAbsolutePath().replaceAll("\\\\","/")+"\"/>\n" +
       "</configuration>\n");
 

Modified: incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BasePostgresql.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BasePostgresql.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BasePostgresql.java (original)
+++ incubator/lcf/trunk/framework/core/src/test/java/org/apache/manifoldcf/core/tests/BasePostgresql.java Mon Oct 31 14:26:33 2011
@@ -99,6 +99,7 @@ public class BasePostgresql
       "  <property name=\"org.apache.manifoldcf.databaseimplementationclass\" value=\"org.apache.manifoldcf.core.database.DBInterfacePostgreSQL\"/>\n" +
       "  <property name=\"org.apache.manifoldcf.database.name\" value=\"testdb\"/>\n" +
       "  <property name=\"org.apache.manifoldcf.database.username\" value=\"testuser\"/>\n" +
+      "  <property name=\"org.apache.manifoldcf.database.maxquerytime\" value=\"30\"/>\n" +
       "  <property name=\"org.apache.manifoldcf.crawler.threads\" value=\"30\"/>\n" +
       "  <property name=\"org.apache.manifoldcf.crawler.expirethreads\" value=\"10\"/>\n" +
       "  <property name=\"org.apache.manifoldcf.crawler.cleanupthreads\" value=\"10\"/>\n" +

Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java?rev=1195471&r1=1195470&r2=1195471&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java (original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java Mon Oct 31 14:26:33 2011
@@ -258,6 +258,7 @@ public class AuthorityConnectionManager 
     StringSet cacheKeys = new StringSet(ssb);
     while (true)
     {
+      long sleepAmt = 0L;
       try
       {
         ICacheHandle ch = cacheManager.enterCache(null,cacheKeys,getTransactionID());
@@ -271,9 +272,10 @@ public class AuthorityConnectionManager 
             boolean isNew = object.getIsNew();
             // See whether the instance exists
             ArrayList params = new ArrayList();
-            params.add(object.getName());
+            String query = buildConjunctionClause(params,new ClauseDescription[]{
+              new UnitaryClause(nameField,object.getName())});
             IResultSet set = performQuery("SELECT * FROM "+getTableName()+" WHERE "+
-              nameField+"=? FOR UPDATE",params,null,null);
+              query+" FOR UPDATE",params,null,null);
             HashMap values = new HashMap();
             values.put(descriptionField,object.getDescription());
             values.put(classNameField,object.getClassName());
@@ -290,8 +292,9 @@ public class AuthorityConnectionManager 
               isCreated = false;
               // Update
               params.clear();
-              params.add(object.getName());
-              performUpdate(values," WHERE "+nameField+"=?",params,null);
+              query = buildConjunctionClause(params,new ClauseDescription[]{
+                new UnitaryClause(nameField,object.getName())});
+              performUpdate(values," WHERE "+query,params,null);
             }
             else
             {
@@ -333,19 +336,16 @@ public class AuthorityConnectionManager 
         // Is this a deadlock exception?  If so, we want to try again.
         if (e.getErrorCode() != ManifoldCFException.DATABASE_TRANSACTION_ABORT)
           throw e;
-        try
-        {
-          ManifoldCF.sleep((long)(random.nextDouble() * 60000.0 + 500.0));
-        }
-        catch (InterruptedException e2)
-        {
-          throw new ManifoldCFException(e2.getMessage(),e2,ManifoldCFException.INTERRUPTED);
-        }
+        sleepAmt = getSleepAmt();
+      }
+      finally
+      {
+        sleepFor(sleepAmt);
       }
     }
   }
 
-  /** Delete a repository connection.
+  /** Delete an authority connection.
   *@param name is the name of the connection to delete.  If the
   * name does not exist, no error is returned.
   */
@@ -370,8 +370,9 @@ public class AuthorityConnectionManager 
           throw new ManifoldCFException("Can't delete authority connection '"+name+"': existing repository connections refer to it");
         ManifoldCF.noteConfigurationChange();
         ArrayList params = new ArrayList();
-        params.add(name);
-        performDelete("WHERE "+nameField+"=?",params,null);
+        String query = buildConjunctionClause(params,new ClauseDescription[]{
+          new UnitaryClause(nameField,name)});
+        performDelete("WHERE "+query,params,null);
         cacheManager.invalidateKeys(ch);
       }
       catch (ManifoldCFException e)
@@ -446,28 +447,23 @@ public class AuthorityConnectionManager 
     try
     {
       i = 0;
-      StringBuilder sb = new StringBuilder();
       ArrayList params = new ArrayList();
       int j = 0;
-      int maxIn = getMaxInClause();
+      int maxIn = maxClauseGetAuthorityConnectionsChunk();
       while (i < connectionNames.length)
       {
         if (j == maxIn)
         {
-          getAuthorityConnectionsChunk(rval,returnIndex,sb.toString(),params);
-          sb.setLength(0);
+          getAuthorityConnectionsChunk(rval,returnIndex,params);
           params.clear();
           j = 0;
         }
-        if (j > 0)
-          sb.append(',');
-        sb.append('?');
         params.add(connectionNames[i]);
         i++;
         j++;
       }
       if (j > 0)
-        getAuthorityConnectionsChunk(rval,returnIndex,sb.toString(),params);
+        getAuthorityConnectionsChunk(rval,returnIndex,params);
       return rval;
     }
     catch (Error e)
@@ -486,18 +482,26 @@ public class AuthorityConnectionManager 
     }
   }
 
+  /** Find the maximum number of clauses for getAuthorityConnectionsChunk.
+  */
+  protected int maxClauseGetAuthorityConnectionsChunk()
+  {
+    return findConjunctionClauseMax(new ClauseDescription[]{});
+  }
+    
   /** Read a chunk of authority connections.
   *@param rval is the place to put the read policies.
   *@param returnIndex is a map from the object id (resource id) and the rval index.
-  *@param idList is the list of id's.
   *@param params is the set of parameters.
   */
-  protected void getAuthorityConnectionsChunk(AuthorityConnection[] rval, Map returnIndex, String idList, ArrayList params)
+  protected void getAuthorityConnectionsChunk(AuthorityConnection[] rval, Map returnIndex, ArrayList params)
     throws ManifoldCFException
   {
-    IResultSet set;
-    set = performQuery("SELECT * FROM "+getTableName()+" WHERE "+
-      nameField+" IN ("+idList+")",params,null,null);
+    ArrayList list = new ArrayList();
+    String query = buildConjunctionClause(list,new ClauseDescription[]{
+      new MultiClause(nameField,params)});
+    IResultSet set = performQuery("SELECT * FROM "+getTableName()+" WHERE "+
+      query,list,null,null);
     int i = 0;
     while (i < set.getRowCount())
     {



Mime
View raw message