manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1023365 - in /incubator/lcf/trunk/modules/framework: agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/ agents/src/main/java/org/apache/manifoldcf/agents/interfaces/ core/src/main/java/org/apache/manifoldcf/core/database/...
Date Sat, 16 Oct 2010 19:45:46 GMT
Author: kwright
Date: Sat Oct 16 19:45:45 2010
New Revision: 1023365

URL: http://svn.apache.org/viewvc?rev=1023365&view=rev
Log:
Change the way database maintenance is done, so that it can be centrally tuned within the database implementation.  Part of CONNECTORS-117.

Modified:
    incubator/lcf/trunk/modules/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
    incubator/lcf/trunk/modules/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java
    incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
    incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
    incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
    incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
    incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java
    incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
    incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopDeleteDeps.java
    incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java
    incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
    incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
    incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/PrereqEventManager.java

Modified: incubator/lcf/trunk/modules/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java?rev=1023365&r1=1023364&r2=1023365&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java (original)
+++ incubator/lcf/trunk/modules/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java Sat Oct 16 19:45:45 2010
@@ -58,8 +58,6 @@ public class IncrementalIngester extends
   // Output connection manager
   protected IOutputConnectionManager connectionManager;
 
-  // Analyze tracker
-  protected static AnalyzeTracker tracker = new AnalyzeTracker();
 
   /** Constructor.
   */
@@ -142,39 +140,6 @@ public class IncrementalIngester extends
 
   }
 
-  /** Come up with a maximum time (in minutes) for re-analyzing tables.
-  *@return the time, in minutes.
-  */
-  public int getAnalyzeTime()
-    throws ManifoldCFException
-  {
-    // For this table, we base the wait time on the number of rows in it.
-    IResultSet set = performQuery("SELECT COUNT("+idField+") FROM "+getTableName(),null,null,null);
-    if (set.getRowCount() < 1)
-      throw new ManifoldCFException("Expected result with one row");
-    IResultRow row = set.getRow(0);
-    Iterator columnNames = row.getColumns();
-    if (!columnNames.hasNext())
-      throw new ManifoldCFException("Expected result with one column");
-    String columnName = (String)columnNames.next();
-    long value = new Long(row.getValue(columnName).toString()).longValue();
-    if (value < 10000L)
-      return 5;
-    else if (value < 100000L)
-      return 2*60;
-    else
-      return 24*60;
-
-  }
-
-  /** Analyze database tables.
-  */
-  public void analyzeTables()
-    throws ManifoldCFException
-  {
-    analyzeTable();
-  }
-
   /** Uninstall the incremental ingestion manager.
   */
   public void deinstall()
@@ -1265,7 +1230,7 @@ public class IncrementalIngester extends
       try
       {
         performInsert(map,null);
-        conditionallyAnalyzeInsert();
+        noteModifications(1,0,0);
         return;
       }
       catch (ManifoldCFException e)
@@ -1454,57 +1419,6 @@ public class IncrementalIngester extends
     return documentClass + ":" + documentHash;
   }
 
-  /** Conditionally do analyze operation.
-  */
-  protected void conditionallyAnalyzeInsert()
-    throws ManifoldCFException
-  {
-    synchronized (tracker)
-    {
-      if (tracker.checkAnalyzeInsert())
-      {
-        // Do the analyze
-        analyzeTable();
-        // Get the size of the table
-        // Simply reanalyze every 8000 inserts
-        tracker.doAnalyze(8000);
-      }
-    }
-  }
-
-  /** Analyze tracker class.
-  */
-  protected static class AnalyzeTracker
-  {
-    // Number of records to insert before we need to analyze again
-    protected long recordCount = 0;
-
-    /** Constructor.
-    */
-    public AnalyzeTracker()
-    {
-
-    }
-
-    /** Note an analyze.
-    */
-    public void doAnalyze(long repeatCount)
-    {
-      recordCount = repeatCount;
-    }
-
-    /** Prepare to insert/delete a record, and see if analyze is required.
-    */
-    public boolean checkAnalyzeInsert()
-    {
-      if (recordCount > 0L)
-        recordCount--;
-      return recordCount == 0L;
-    }
-
-
-  }
-
   /** This class contains the information necessary to delete a document */
   protected static class DeleteInfo
   {

Modified: incubator/lcf/trunk/modules/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java?rev=1023365&r1=1023364&r2=1023365&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java (original)
+++ incubator/lcf/trunk/modules/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IIncrementalIngester.java Sat Oct 16 19:45:45 2010
@@ -53,17 +53,6 @@ public interface IIncrementalIngester
   public void deinstall()
     throws ManifoldCFException;
 
-  /** Come up with a maximum time (in minutes) for re-analyzing tables.
-  *@return the time, in minutes.
-  */
-  public int getAnalyzeTime()
-    throws ManifoldCFException;
-
-  /** Analyze database tables.
-  */
-  public void analyzeTables()
-    throws ManifoldCFException;
-
   /** Flush all knowledge of what was ingested before.
   */
   public void clearAll()

Modified: incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java?rev=1023365&r1=1023364&r2=1023365&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java (original)
+++ incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java Sat Oct 16 19:45:45 2010
@@ -274,6 +274,17 @@ public class BaseTable
     dbInterface.endTransaction();
   }
 
+  /** Note a number of inserts, modifications, or deletions to a specific table.  This is so we can decide when to do appropriate maintenance.
+  *@param tableName is the name of the table being modified.
+  *@param insertCount is the number of inserts.
+  *@param modifyCount is the number of updates.
+  *@param deleteCount is the number of deletions.
+  */
+  public void noteModifications(int insertCount, int modifyCount, int deleteCount)
+    throws ManifoldCFException
+  {
+    dbInterface.noteModifications(tableName,insertCount,modifyCount,deleteCount);
+  }
 
   /** Construct a key that is database specific, and applies to queries
   * made against a specific table name.

Modified: incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java?rev=1023365&r1=1023364&r2=1023365&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java (original)
+++ incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java Sat Oct 16 19:45:45 2010
@@ -448,10 +448,7 @@ public class DBInterfacePostgreSQL exten
     performModification("DROP INDEX "+indexName,null,null);
   }
 
-  /** Analyze a table.
-  *@param tableName is the name of the table to analyze/calculate statistics for.
-  */
-  public void analyzeTable(String tableName)
+  protected void analyzeTableInternal(String tableName)
     throws ManifoldCFException
   {
     if (getTransactionID() == null)
@@ -460,14 +457,34 @@ public class DBInterfacePostgreSQL exten
       tablesToAnalyze.add(tableName);
   }
 
-  /** Reindex a table.
-  *@param tableName is the name of the table to rebuild indexes for.
-  */
-  public void reindexTable(String tableName)
+  protected void reindexTableInternal(String tableName)
     throws ManifoldCFException
   {
     if (getTransactionID() == null)
-      performModification("REINDEX TABLE "+tableName,null,null);
+    {
+      long sleepAmt = 0L;
+      while (true)
+      {
+        try
+        {
+          performModification("REINDEX TABLE "+tableName,null,null);
+          break;
+        }
+        catch (ManifoldCFException e)
+        {
+          if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
+          {
+            sleepAmt = getSleepAmt();
+            continue;
+          }
+          throw e;
+        }
+        finally
+        {
+          sleepFor(sleepAmt);
+        }
+      }
+    }
     else
       tablesToReindex.add(tableName);
   }
@@ -1049,13 +1066,13 @@ public class DBInterfacePostgreSQL exten
       int i = 0;
       while (i < tablesToAnalyze.size())
       {
-        analyzeTable((String)tablesToAnalyze.get(i++));
+        analyzeTableInternal((String)tablesToAnalyze.get(i++));
       }
       tablesToAnalyze.clear();
       i = 0;
       while (i < tablesToReindex.size())
       {
-        reindexTable((String)tablesToReindex.get(i++));
+        reindexTableInternal((String)tablesToReindex.get(i++));
       }
       tablesToReindex.clear();
     }
@@ -1140,6 +1157,85 @@ public class DBInterfacePostgreSQL exten
     lockManager.writeData(datumName,bytes);
   }
 
+  // Lock and shared datum name prefixes (to be combined with table names)
+  protected static final String statslockReindexPrefix = "statslock-reindex-";
+  protected static final String statsReindexPrefix = "stats-reindex-";
+  protected static final String statslockAnalyzePrefix = "statslock-analyze-";
+  protected static final String statsAnalyzePrefix = "stats-analyze-";
+  
+  /** Analyze a table.
+  *@param tableName is the name of the table to analyze/calculate statistics for.
+  */
+  public void analyzeTable(String tableName)
+    throws ManifoldCFException
+  {
+    String tableStatisticsLock = statslockAnalyzePrefix+tableName;
+    lockManager.enterWriteCriticalSection(tableStatisticsLock);
+    try
+    {
+      TableStatistics ts = (TableStatistics)currentAnalyzeStatistics.get(tableName);
+      // Lock this table's statistics files
+      lockManager.enterWriteLock(tableStatisticsLock);
+      try
+      {
+        String eventDatum = statsAnalyzePrefix+tableName;
+        // Time to reindex this table!
+        analyzeTableInternal(tableName);
+        // Now, clear out the data
+        writeDatum(eventDatum,0);
+        if (ts != null)
+          ts.reset();
+      }
+      finally
+      {
+        lockManager.leaveWriteLock(tableStatisticsLock);
+      }
+    }
+    finally
+    {
+      lockManager.leaveWriteCriticalSection(tableStatisticsLock);
+    }
+
+    analyzeTableInternal(tableName);
+  }
+
+  /** Reindex a table.
+  *@param tableName is the name of the table to rebuild indexes for.
+  */
+  public void reindexTable(String tableName)
+    throws ManifoldCFException
+  {
+    String tableStatisticsLock;
+    
+    // Reindexing.
+    tableStatisticsLock = statslockReindexPrefix+tableName;
+    lockManager.enterWriteCriticalSection(tableStatisticsLock);
+    try
+    {
+      TableStatistics ts = (TableStatistics)currentReindexStatistics.get(tableName);
+      // Lock this table's statistics files
+      lockManager.enterWriteLock(tableStatisticsLock);
+      try
+      {
+        String eventDatum = statsReindexPrefix+tableName;
+        // Time to reindex this table!
+        reindexTableInternal(tableName);
+        // Now, clear out the data
+        writeDatum(eventDatum,0);
+        if (ts != null)
+          ts.reset();
+      }
+      finally
+      {
+        lockManager.leaveWriteLock(tableStatisticsLock);
+      }
+    }
+    finally
+    {
+      lockManager.leaveWriteCriticalSection(tableStatisticsLock);
+    }
+  }
+
   /** Note a number of inserts, modifications, or deletions to a specific table.  This is so we can decide when to do appropriate maintenance.
   *@param tableName is the name of the table being modified.
   *@param insertCount is the number of inserts.
@@ -1155,7 +1251,7 @@ public class DBInterfacePostgreSQL exten
     // Reindexing.
     // Here we count tuple deletion.  So we want to know the deletecount + modifycount.
     eventCount = modifyCount + deleteCount;
-    tableStatisticsLock = "statslock-reindex-"+tableName;
+    tableStatisticsLock = statslockReindexPrefix+tableName;
     lockManager.enterWriteCriticalSection(tableStatisticsLock);
     try
     {
@@ -1164,7 +1260,7 @@ public class DBInterfacePostgreSQL exten
       if (threshold == null)
       {
         // Look for this parameter; if we don't find it, use a default value.
-        reindexThreshold = ManifoldCF.getIntProperty("org.apache.manifold.db.postgres.reindex."+tableName,100000);
+        reindexThreshold = ManifoldCF.getIntProperty("org.apache.manifold.db.postgres.reindex."+tableName,250000);
         reindexThresholds.put(tableName,new Integer(reindexThreshold));
       }
       else
@@ -1184,13 +1280,13 @@ public class DBInterfacePostgreSQL exten
         lockManager.enterWriteLock(tableStatisticsLock);
         try
         {
-          String eventDatum = "stats-reindex-"+tableName;
+          String eventDatum = statsReindexPrefix+tableName;
           int oldEventCount = readDatum(eventDatum);
           oldEventCount += ts.getEventCount();
           if (oldEventCount >= reindexThreshold)
           {
             // Time to reindex this table!
-            reindexTable(tableName);
+            reindexTableInternal(tableName);
             // Now, clear out the data
             writeDatum(eventDatum,0);
           }
@@ -1212,7 +1308,7 @@ public class DBInterfacePostgreSQL exten
       // Analysis.
     // Here we count tuple addition.
     eventCount = modifyCount + insertCount;
-    tableStatisticsLock = "statslock-analyze-"+tableName;
+    tableStatisticsLock = statslockAnalyzePrefix+tableName;
     lockManager.enterWriteCriticalSection(tableStatisticsLock);
     try
     {
@@ -1241,13 +1337,13 @@ public class DBInterfacePostgreSQL exten
         lockManager.enterWriteLock(tableStatisticsLock);
         try
         {
-          String eventDatum = "stats-analyze-"+tableName;
+          String eventDatum = statsAnalyzePrefix+tableName;
           int oldEventCount = readDatum(eventDatum);
           oldEventCount += ts.getEventCount();
           if (oldEventCount >= analyzeThreshold)
           {
             // Time to reindex this table!
-            analyzeTable(tableName);
+            analyzeTableInternal(tableName);
             // Now, clear out the data
             writeDatum(eventDatum,0);
           }
@@ -1268,6 +1364,7 @@ public class DBInterfacePostgreSQL exten
 
   }
   
+
   /** Table accumulation records.
   */
   protected static class TableStatistics

Modified: incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java?rev=1023365&r1=1023364&r2=1023365&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java (original)
+++ incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java Sat Oct 16 19:45:45 2010
@@ -20,6 +20,7 @@ package org.apache.manifoldcf.core.datab
 
 import org.apache.manifoldcf.core.interfaces.*;
 import org.apache.manifoldcf.core.system.Logging;
+import org.apache.manifoldcf.core.system.ManifoldCF;
 
 import java.util.*;
 import java.sql.*;
@@ -354,6 +355,34 @@ public class Database
   {
   }
 
+  protected static Random random = new Random();
+
+  /** Sleep a random amount of time after a transaction abort.
+  */
+  public long getSleepAmt()
+  {
+    // Amount should be between .5 and 1 minute, approx, to give things time to unwind
+    return (long)(random.nextDouble() * 60000.0 + 500.0);
+  }
+
+  /** Sleep, as part of recovery from deadlock.
+  */
+  public void sleepFor(long amt)
+    throws ManifoldCFException
+  {
+    if (amt == 0L)
+      return;
+
+    try
+    {
+      ManifoldCF.sleep(amt);
+    }
+    catch (InterruptedException e)
+    {
+      throw new ManifoldCFException("Interrupted",e,ManifoldCFException.INTERRUPTED);
+    }
+  }
+
   /** Thread used to execute queries.  An instance of this thread is spun up every time a query is executed.  This is necessary because JDBC does not
   * guarantee interruptability, and the Postgresql JDBC driver unfortunately eats all thread interrupts.  So, we fire up a thread to do each interaction with
   * the database server, thus insuring that the owning thread remains interruptable and will therefore not block shutdown.

Modified: incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java?rev=1023365&r1=1023364&r2=1023365&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java (original)
+++ incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java Sat Oct 16 19:45:45 2010
@@ -364,5 +364,16 @@ public interface IDBInterface
   public void noteModifications(String tableName, int insertCount, int modifyCount, int deleteCount)
     throws ManifoldCFException;
 
+  /** Get a random time, in milliseconds, for backoff from deadlock.
+  *@return the random time.
+  */
+  public long getSleepAmt();
+  
+  /** Sleep for a specified time, as part of backoff from deadlock.
+  *@param time is the amount to sleep.
+  */
+  public void sleepFor(long time)
+    throws ManifoldCFException;
+    
 }
 

Modified: incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java?rev=1023365&r1=1023364&r2=1023365&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java (original)
+++ incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Carrydown.java Sat Oct 16 19:45:45 2010
@@ -48,9 +48,6 @@ public class Carrydown extends org.apach
   /** This value means that the link existed before, and has been found during this scan. */
   protected static final int ISNEW_EXISTING = 2;
 
-  /** Counter for kicking off analyze */
-  protected static AnalyzeTracker tracker = new AnalyzeTracker();
-
   // Map from string character to link status
   protected static Map isNewMap;
   static
@@ -341,7 +338,7 @@ public class Carrydown extends org.apach
 
         map.put(newField,statusToString(ISNEW_NEW));
         performInsert(map,null);
-        tracker.noteInsert();
+        noteModifications(1,0,0);
         insertHappened.put(childDocumentIDHash,new Boolean(true));
       }
       else
@@ -368,6 +365,7 @@ public class Carrydown extends org.apach
 
         map.put(newField,statusToString(ISNEW_EXISTING));
         performUpdate(map,sb.toString(),updateList,null);
+        noteModifications(0,1,0);
       }
     }
 
@@ -472,6 +470,7 @@ public class Carrydown extends org.apach
     HashMap map = new HashMap();
     map.put(newField,statusToString(ISNEW_BASE));
     performUpdate(map,sb.toString(),list,null);
+    noteModifications(0,list.size(),0);
   }
 
   /** Delete all records that mention a particular set of document identifiers.
@@ -546,6 +545,7 @@ public class Carrydown extends org.apach
   {
     performDelete("WHERE "+query,list,null);
     performDelete("WHERE "+query2,list2,null);
+    noteModifications(0,0,list.size()+list2.size());
   }
 
   /** Get unique values given a document identifier, data name, an job identifier */
@@ -621,28 +621,6 @@ public class Carrydown extends org.apach
     }
   }
 
-  /** Conditionally do analyze operation.
-  */
-  public void conditionallyAnalyzeTables()
-    throws ManifoldCFException
-  {
-    if (tracker.checkAnalyze())
-    {
-      try
-      {
-        // Do the analyze
-        analyzeTable();
-      }
-      finally
-      {
-        // Get the size of the table
-        // For this table, we base the wait time on the number of rows in it.
-        // Simply reanalyze every n inserts
-        tracker.doAnalyze(30000L);
-      }
-    }
-  }
-
   /** Limit checker which removes duplicate rows, based on datavaluehash */
   protected static class ResultDuplicateEliminator implements ILimitChecker
   {
@@ -702,50 +680,6 @@ public class Carrydown extends org.apach
     }
   }
 
-  /** Analyze tracker class.
-  */
-  protected static class AnalyzeTracker
-  {
-    // Number of records to insert before we need to analyze again.
-    // After start, we wait 1000 before analyzing the first time.
-    protected long recordCount = 1000L;
-    protected boolean busy = false;
-
-    /** Constructor.
-    */
-    public AnalyzeTracker()
-    {
-
-    }
-
-    /** Note an analyze.
-    */
-    public synchronized void doAnalyze(long repeatCount)
-    {
-      recordCount = repeatCount;
-      busy = false;
-    }
-
-    /** Note an insert */
-    public synchronized void noteInsert()
-    {
-      if (recordCount > 0L)
-        recordCount--;
-    }
-
-    /** Prepare to insert/delete a record, and see if analyze is required.
-    */
-    public synchronized boolean checkAnalyze()
-    {
-      if (busy)
-        return false;
-      busy = (recordCount == 0L);
-      return busy;
-    }
-
-
-  }
-
   protected static class ValueRecord
   {
     protected String documentIdentifierHash;

Modified: incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java?rev=1023365&r1=1023364&r2=1023365&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java (original)
+++ incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopCount.java Sat Oct 16 19:45:45 2010
@@ -125,14 +125,6 @@ public class HopCount extends org.apache
     markMap.put("D",new Integer(MARK_DELETING));
   }
 
-  /** Counter for kicking off analyze */
-  protected static AnalyzeTracker tracker = new AnalyzeTracker();
-  /** Counter for kicking off reindex */
-  protected static AnalyzeTracker reindexTracker = new AnalyzeTracker();
-
-  // The number of updates before doing a reindex
-  protected static long REINDEX_COUNT = 250000L;
-
   /** Intrinsic link table manager. */
   protected IntrinsicLink intrinsicLinkManager;
   /** Hop "delete" dependencies manager */
@@ -299,7 +291,7 @@ public class HopCount extends org.apache
       ArrayList list = new ArrayList();
       list.add(jobID);
       performDelete("WHERE "+jobIDField+"=?",list,null);
-      reindexTracker.noteInsert();
+      noteModifications(0,0,1);
     }
     catch (ManifoldCFException e)
     {
@@ -893,7 +885,7 @@ public class HopCount extends org.apache
             if (Logging.hopcount.isDebugEnabled())
               Logging.hopcount.debug("Inserting new record for '"+documentIDHash+"' linktype '"+affectedLinkType+"' distance "+Integer.toString(newAnswerValue)+" for job "+jobID);
             performInsert(map,null);
-            tracker.noteInsert();
+            noteModifications(1,0,0);
             if (hopcountMethod != IJobDescription.HOPCOUNT_NEVERDELETE)
             {
               deleteDepsManager.writeDependency(hopCountID,jobID,dd);
@@ -997,7 +989,7 @@ public class HopCount extends org.apache
     {
       endTransaction();
     }
-    reindexTracker.noteInsert(documentIDHashes.length);
+    noteModifications(0,documentIDHashes.length,0);
   }
 
   /** Do the work of marking add-dep-dependent links in the hopcount table. */
@@ -1137,7 +1129,7 @@ public class HopCount extends org.apache
         }
         if (k > 0)
           markForDelete(sb.toString(),list,commonNewExpression);
-        reindexTracker.noteInsert(sourceDocumentHashes.length);
+        noteModifications(0,sourceDocumentHashes.length,0);
       }
       else
       {
@@ -1174,7 +1166,7 @@ public class HopCount extends org.apache
         map.put(distanceField,new Long(-1L));
         map.put(markForDeathField,markToString(MARK_DELETING));
         performUpdate(map,sb.toString(),list,null);
-        reindexTracker.noteInsert();
+        noteModifications(0,1,0);
       }
 
       if (Logging.hopcount.isDebugEnabled())
@@ -1550,7 +1542,7 @@ public class HopCount extends org.apache
           ArrayList list = new ArrayList();
           list.add(existingID);
           performDelete("WHERE "+idField+"=?",list,null);
-          reindexTracker.noteInsert();
+          noteModifications(0,0,1);
           // Since infinity is not a reduction of any kind, we're done here.
           return;
         }
@@ -1638,7 +1630,7 @@ public class HopCount extends org.apache
           ArrayList list = new ArrayList();
           list.add(existingID);
           performUpdate(map,"WHERE "+idField+"=?",list,null);
-          reindexTracker.noteInsert();
+          noteModifications(0,1,0);
 
           if (hopcountMethod != IJobDescription.HOPCOUNT_NEVERDELETE)
           {
@@ -1719,8 +1711,7 @@ public class HopCount extends org.apache
         list.add(existingID);
         map.put(markForDeathField,markToString(MARK_NORMAL));
         performUpdate(map,"WHERE "+idField+"=?",list,null);
-        tracker.noteInsert();
-        reindexTracker.noteInsert();
+        noteModifications(0,1,0);
       }
 
       // Done
@@ -1757,7 +1748,7 @@ public class HopCount extends org.apache
       map.put(parentIDHashField,parentIDHash);
       map.put(distanceField,new Long(answer.getAnswer()));
       performInsert(map,null);
-      tracker.noteInsert();
+      noteModifications(1,0,0);
 
       if (hopcountMethod != IJobDescription.HOPCOUNT_NEVERDELETE)
       {
@@ -1786,98 +1777,6 @@ public class HopCount extends org.apache
     }
   }
 
-  /** Conditionally do analyze operation.
-  */
-  public void conditionallyAnalyzeTables()
-    throws ManifoldCFException
-  {
-    if (tracker.checkAnalyze())
-    {
-      try
-      {
-        // Do the analyze
-        analyzeTable();
-        // For this table, we base the wait time on the number of rows in it.
-        // Simply reanalyze every n inserts
-      }
-      finally
-      {
-        tracker.doAnalyze(30000L);
-      }
-    }
-    if (reindexTracker.checkAnalyze())
-    {
-      try
-      {
-        // Do the reindex
-        reindexTable();
-        // For this table, we base the wait time on the number of rows in it.
-        // Simply reanalyze every n inserts
-      }
-      finally
-      {
-        reindexTracker.doAnalyze(REINDEX_COUNT);
-      }
-    }
-    intrinsicLinkManager.conditionallyAnalyzeTables();
-    deleteDepsManager.conditionallyAnalyzeTables();
-  }
-
-
-  /** Analyze tracker class.
-  */
-  protected static class AnalyzeTracker
-  {
-    // Number of records to insert before we need to analyze again.
-    // After start, we wait 1000 before analyzing the first time.
-    protected long recordCount = 1000L;
-    protected boolean busy = false;
-
-    /** Constructor.
-    */
-    public AnalyzeTracker()
-    {
-
-    }
-
-    /** Note an analyze.
-    */
-    public synchronized void doAnalyze(long repeatCount)
-    {
-      recordCount = repeatCount;
-      busy = false;
-    }
-
-    public synchronized void noteInsert(int count)
-    {
-      if (recordCount >= (long)count)
-        recordCount -= (long)count;
-      else
-        recordCount = 0L;
-    }
-
-    /** Note an insert */
-    public synchronized void noteInsert()
-    {
-      if (recordCount > 0L)
-        recordCount--;
-    }
-
-    /** Prepare to insert/delete a record, and see if analyze is required.
-    */
-    public synchronized boolean checkAnalyze()
-    {
-      if (busy)
-        return false;
-      busy = (recordCount == 0L);
-      return busy;
-    }
-
-
-  }
-
-
-
   /** A class describing a document identifier and a link type, to be used in looking up the appropriate node in
   * the hash.
   */

Modified: incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopDeleteDeps.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopDeleteDeps.java?rev=1023365&r1=1023364&r2=1023365&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopDeleteDeps.java (original)
+++ incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/HopDeleteDeps.java Sat Oct 16 19:45:45 2010
@@ -38,14 +38,6 @@ public class HopDeleteDeps extends org.a
   public static final String parentIDHashField = "parentidhash";
   public static final String childIDHashField = "childidhash";
 
-  /** Counter for kicking off analyze */
-  protected static AnalyzeTracker tracker = new AnalyzeTracker();
-  /** Counter for kicking off reindex */
-  protected static AnalyzeTracker reindexTracker = new AnalyzeTracker();
-
-  // Number of events before reindex occurs
-  protected static final long REINDEX_COUNT = 250000L;
-
   /** Constructor.
   *@param database is the database handle.
   */
@@ -148,7 +140,7 @@ public class HopDeleteDeps extends org.a
     list.add(jobID);
     performDelete("WHERE "+jobIDField+"=?",list,null);
     // Log one event - it may not be enough, but it's the best we can do without overhead
-    reindexTracker.noteInsert();
+    noteModifications(0,0,1);
   }
 
   /** Remove rows that correspond to specific hopcount records.
@@ -162,7 +154,7 @@ public class HopDeleteDeps extends org.a
     performDelete("WHERE "+ownerIDField+" IN(SELECT "+parentIDHashField+" FROM "+parentTable+" WHERE "+query+")",
       queryList,null);
     // Log one event - it may not be enough, but it's the best we can do without overhead
-    reindexTracker.noteInsert();
+    noteModifications(0,0,1);
   }
 
   /** Delete rows related to specified owners.  The list of
@@ -185,7 +177,7 @@ public class HopDeleteDeps extends org.a
     }
     sb.append(")");
     performDelete(sb.toString(),list,null);
-    reindexTracker.noteInsert(ownerIDs.length);
+    noteModifications(0,0,ownerIDs.length);
   }
 
   /** Get the delete dependencies for an owner.
@@ -236,7 +228,7 @@ public class HopDeleteDeps extends org.a
     else
       sb.append(childIDHashField).append(" IS NULL");
     performDelete(sb.toString(),list,null);
-    reindexTracker.noteInsert();
+    noteModifications(0,0,1);
   }
 
   /** Write a delete dependency.
@@ -255,101 +247,7 @@ public class HopDeleteDeps extends org.a
       map.put(childIDHashField,dd.getChildIDHash());
     }
     performInsert(map,null);
-    tracker.noteInsert();
-  }
-
-
-  /** Conditionally do analyze operation.
-  */
-  public void conditionallyAnalyzeTables()
-    throws ManifoldCFException
-  {
-    if (tracker.checkAnalyze())
-    {
-      try
-      {
-        // Do the analyze
-        analyzeTable();
-        // Get the size of the table
-      }
-      finally
-      {
-        // For this table, we base the wait time on the number of rows in it.
-        // Simply reanalyze every n inserts
-        tracker.doAnalyze(60000L);
-      }
-    }
-    if (reindexTracker.checkAnalyze())
-    {
-      try
-      {
-        // Do the reindex
-        reindexTable();
-        // Get the size of the table
-      }
-      finally
-      {
-        // For this table, we base the wait time on the number of rows in it.
-        // Simply reanalyze every n inserts
-        reindexTracker.doAnalyze(REINDEX_COUNT);
-      }
-    }
-
-  }
-
-
-  /** Analyze tracker class.
-  */
-  protected static class AnalyzeTracker
-  {
-    // Number of records to insert before we need to analyze again.
-    // After start, we wait 1000 before analyzing the first time.
-    protected long recordCount = 1000L;
-    protected boolean busy = false;
-
-    /** Constructor.
-    */
-    public AnalyzeTracker()
-    {
-
-    }
-
-    /** Note an analyze.
-    */
-    public synchronized void doAnalyze(long repeatCount)
-    {
-      recordCount = repeatCount;
-      busy = false;
-    }
-
-    public synchronized void noteInsert(int count)
-    {
-      if (recordCount >= (long)count)
-        recordCount -= (long)count;
-      else
-        recordCount = 0L;
-    }
-
-    /** Note an insert */
-    public synchronized void noteInsert()
-    {
-      if (recordCount > 0L)
-        recordCount--;
-    }
-
-    /** Prepare to insert/delete a record, and see if analyze is required.
-    */
-    public synchronized boolean checkAnalyze()
-    {
-      if (busy)
-        return false;
-      busy = (recordCount == 0L);
-      return busy;
-    }
-
-
+    noteModifications(1,0,0);
   }
 
-
-
 }

Modified: incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java?rev=1023365&r1=1023364&r2=1023365&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java (original)
+++ incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/IntrinsicLink.java Sat Oct 16 19:45:45 2010
@@ -58,14 +58,6 @@ public class IntrinsicLink extends org.a
     linkstatusMap.put("E",new Integer(LINKSTATUS_EXISTING));
   }
 
-  /** Counter for kicking off analyze */
-  protected static AnalyzeTracker tracker = new AnalyzeTracker();
-  /** Counter for kicking off reindex */
-  protected static AnalyzeTracker reindexTracker = new AnalyzeTracker();
-
-  // Count of events to call reindex for
-  protected static final long REINDEX_COUNT = 250000L;
-
   /** Constructor.
   *@param database is the database handle.
   */
@@ -165,7 +157,7 @@ public class IntrinsicLink extends org.a
     ArrayList list = new ArrayList();
     list.add(jobID);
     performDelete("WHERE "+jobIDField+"=?",list,null);
-    reindexTracker.noteInsert();
+    noteModifications(0,0,1);
   }
 
   /** Reset, at startup time.  Since links can only be added in a transactionally safe way by processing
@@ -262,7 +254,7 @@ public class IntrinsicLink extends org.a
           map.put(linkTypeField,linkType);
           map.put(newField,statusToString(LINKSTATUS_NEW));
           performInsert(map,null);
-          tracker.noteInsert();
+          noteModifications(1,0,0);
         }
         else
         {
@@ -275,7 +267,7 @@ public class IntrinsicLink extends org.a
           map.put(newField,statusToString(LINKSTATUS_EXISTING));
           performUpdate(map,"WHERE "+jobIDField+"=? AND "+linkTypeField+"=? AND "+
             parentIDHashField+"=? AND "+childIDHashField+"=?",updateList,null);
-          reindexTracker.noteInsert();
+          noteModifications(0,1,0);
         }
       }
       return newReferences;
@@ -350,7 +342,7 @@ public class IntrinsicLink extends org.a
 
         if (k > 0)
           performRemoveLinks(sb.toString(),list,commonNewExpression);
-        reindexTracker.noteInsert(sourceDocumentIDHashes.length);
+        noteModifications(0,0,sourceDocumentIDHashes.length);
       }
       else
       {
@@ -361,7 +353,7 @@ public class IntrinsicLink extends org.a
           .append(sourceTableIDColumn).append("=").append(getTableName()).append(".").append(childIDHashField)
           .append(" AND ").append(sourceTableCriteria).append(")");
         performDelete(sb.toString(),list,null);
-        reindexTracker.noteInsert();
+        noteModifications(0,0,1);
       }
     }
     catch (ManifoldCFException e)
@@ -440,7 +432,7 @@ public class IntrinsicLink extends org.a
     {
       endTransaction();
     }
-    reindexTracker.noteInsert(sourceDocumentIDHashes.length);
+    noteModifications(0,sourceDocumentIDHashes.length,0);
   }
 
   protected void performRestoreLinks(String query, ArrayList list)
@@ -512,97 +504,6 @@ public class IntrinsicLink extends org.a
     }
   }
 
-  /** Conditionally do analyze operation.
-  */
-  public void conditionallyAnalyzeTables()
-    throws ManifoldCFException
-  {
-    if (tracker.checkAnalyze())
-    {
-      try
-      {
-        // Do the analyze
-        analyzeTable();
-      }
-      finally
-      {
-        // Get the size of the table
-        // For this table, we base the wait time on the number of rows in it.
-        // Simply reanalyze every n inserts
-        tracker.doAnalyze(30000L);
-      }
-    }
-    if (reindexTracker.checkAnalyze())
-    {
-      try
-      {
-        // Do the reindex
-        reindexTable();
-      }
-      finally
-      {
-        // Get the size of the table
-        // For this table, we base the wait time on the number of rows in it.
-        // Simply reanalyze every n inserts
-        reindexTracker.doAnalyze(REINDEX_COUNT);
-      }
-    }
-
-  }
-
-
-  /** Analyze tracker class.
-  */
-  protected static class AnalyzeTracker
-  {
-    // Number of records to insert before we need to analyze again.
-    // After start, we wait 1000 before analyzing the first time.
-    protected long recordCount = 1000L;
-    protected boolean busy = false;
-
-    /** Constructor.
-    */
-    public AnalyzeTracker()
-    {
-
-    }
-
-    /** Note an analyze.
-    */
-    public synchronized void doAnalyze(long repeatCount)
-    {
-      recordCount = repeatCount;
-      busy = false;
-    }
-
-    public synchronized void noteInsert(int count)
-    {
-      if (recordCount >= (long)count)
-        recordCount -= (long)count;
-      else
-        recordCount = 0L;
-    }
-
-    /** Note an insert */
-    public synchronized void noteInsert()
-    {
-      if (recordCount > 0L)
-        recordCount--;
-    }
-
-    /** Prepare to insert/delete a record, and see if analyze is required.
-    */
-    public synchronized boolean checkAnalyze()
-    {
-      if (busy)
-        return false;
-      busy = (recordCount == 0L);
-      return busy;
-    }
-
-
-  }
-
   // This class filters an ordered resultset to return only the duplicates
   protected static class DuplicateFinder implements ILimitChecker
   {

Modified: incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1023365&r1=1023364&r2=1023365&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java (original)
+++ incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java Sat Oct 16 19:45:45 2010
@@ -677,7 +677,6 @@ public class JobManager implements IJobM
         sleepFor(sleepAmt);
       }
     }
-    conditionallyAnalyzeTables();
   }
 
   /** Reset as part of restoring document worker threads.
@@ -719,7 +718,6 @@ public class JobManager implements IJobM
       }
     }
     Logging.jobs.debug("Reset complete");
-    conditionallyAnalyzeTables();
   }
 
   /** Reset as part of restoring seeding threads.
@@ -730,7 +728,6 @@ public class JobManager implements IJobM
     Logging.jobs.debug("Resetting seeding status");
     jobs.resetSeedingWorkerStatus();
     Logging.jobs.debug("Reset complete");
-    conditionallyAnalyzeTables();
   }
 
   /** Reset as part of restoring doc delete threads.
@@ -741,7 +738,6 @@ public class JobManager implements IJobM
     Logging.jobs.debug("Resetting doc deleting status");
     jobQueue.resetDocDeleteWorkerStatus();
     Logging.jobs.debug("Reset complete");
-    conditionallyAnalyzeTables();
   }
 
   /** Reset as part of restoring startup threads.
@@ -752,7 +748,6 @@ public class JobManager implements IJobM
     Logging.jobs.debug("Resetting job starting up status");
     jobs.resetStartupWorkerStatus();
     Logging.jobs.debug("Reset complete");
-    conditionallyAnalyzeTables();
   }
 
   // These methods support job delete threads
@@ -1270,7 +1265,6 @@ public class JobManager implements IJobM
         sleepFor(sleepAmt);
       }
     }
-    conditionallyAnalyzeTables();
   }
 
   /** Get up to the next n documents to be expired.
@@ -1687,7 +1681,6 @@ public class JobManager implements IJobM
       rval[i] = (DocumentDescription)answers.get(i);
       i++;
     }
-    conditionallyAnalyzeTables();
 
     // After we're done pulling stuff from the queue, find the eligible row with the best priority on the queue, and save the bins for assessment.
     // This done to decide what the "floor" bincount should be - the idea being that it is wrong to assign priorities for new documents which are
@@ -2054,8 +2047,6 @@ public class JobManager implements IJobM
         sleepFor(sleepAmt);
       }
     }
-
-    conditionallyAnalyzeTables();
   }
 
   /** Note completion of document processing by a job thread of a document.
@@ -2170,7 +2161,6 @@ public class JobManager implements IJobM
         sleepFor(sleepAmt);
       }
     }
-    conditionallyAnalyzeTables();
     return rval;
   }
 
@@ -2932,7 +2922,6 @@ public class JobManager implements IJobM
         sleepFor(sleepAmt);
       }
     }
-    conditionallyAnalyzeTables();
   }
 
   /** Get the specified hop counts, with the limit as described.
@@ -3540,7 +3529,6 @@ public class JobManager implements IJobM
         }
       }
     }
-    conditionallyAnalyzeTables();
     return rval;
   }
 
@@ -3762,7 +3750,6 @@ public class JobManager implements IJobM
         sleepFor(sleepAmt);
       }
     }
-    conditionallyAnalyzeTables();
     return rval;
   }
 
@@ -4660,7 +4647,6 @@ public class JobManager implements IJobM
     {
       Logging.jobs.debug("Job "+jobID+" successfully restarted");
     }
-    conditionallyAnalyzeTables();
   }
 
   /** Get the list of jobs that are ready for seeding.
@@ -5174,39 +5160,6 @@ public class JobManager implements IJobM
     }
   }
 
-  /** Conditionally analyze tables.  Each table keeps track of the activity that has occurred on it and
-  * decides if it is time to issue an ANALYZE request.
-  */
-  protected void conditionallyAnalyzeTables()
-    throws ManifoldCFException
-  {
-    while (true)
-    {
-      long sleepAmt = 0L;
-      try
-      {
-        hopCount.conditionallyAnalyzeTables();
-        jobQueue.conditionallyAnalyzeTables();
-        break;
-      }
-      catch (ManifoldCFException e)
-      {
-        if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
-        {
-          if (Logging.perf.isDebugEnabled())
-            Logging.perf.debug("Aborted reindexing job tables: "+e.getMessage());
-          sleepAmt = getRandomAmount();
-          continue;
-        }
-        throw e;
-      }
-      finally
-      {
-        sleepFor(sleepAmt);
-      }
-    }
-  }
-
   /** Put all eligible jobs in the "shutting down" state.
   */
   public void finishJobs()

Modified: incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java?rev=1023365&r1=1023364&r2=1023365&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java (original)
+++ incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java Sat Oct 16 19:45:45 2010
@@ -119,16 +119,6 @@ public class JobQueue extends org.apache
     actionMap.put("D",new Integer(ACTION_REMOVE));
   }
 
-  /** Counter for kicking off analyze */
-  protected static AnalyzeTracker tracker = new AnalyzeTracker();
-  /** Counter for kicking off reindex */
-  protected static AnalyzeTracker reindexTracker = new AnalyzeTracker();
-
-  /** The number of operations before we do a reindex.
-  * Note well: 50,000 worked ok, but reindex was a bit frequent.  100,000 was infrequent enough
-  * with 7.4 to cause occasional stuffing queries to take more than a minute.  Trying 250,000 now (on 8.2) */
-  protected final static long REINDEX_COUNT = 250000L;
-
   /** Prerequisite event manager */
   protected PrereqEventManager prereqEventManager;
 
@@ -256,43 +246,10 @@ public class JobQueue extends org.apache
   public void unconditionallyAnalyzeTables()
     throws ManifoldCFException
   {
-    try
-    {
-      long startTime = System.currentTimeMillis();
-      Logging.perf.debug("Beginning to analyze jobqueue table");
-      analyzeTable();
-      Logging.perf.debug("Done analyzing jobqueue table in "+new Long(System.currentTimeMillis()-startTime)+" ms");
-    }
-    finally
-    {
-      tracker.doAnalyze(15000L);
-    }
-  }
-
-  /** Analyze job tables that need analysis.
-  */
-  public void conditionallyAnalyzeTables()
-    throws ManifoldCFException
-  {
-    if (tracker.checkAnalyze())
-    {
-      unconditionallyAnalyzeTables();
-    }
-    if (reindexTracker.checkAnalyze())
-    {
-      try
-      {
-        long startTime = System.currentTimeMillis();
-        Logging.perf.debug("Beginning to reindex jobqueue table");
-        reindexTable();
-        Logging.perf.debug("Done reindexing jobqueue table in "+new Long(System.currentTimeMillis()-startTime)+" ms");
-      }
-      finally
-      {
-        // Simply reindex every n inserts
-        reindexTracker.doAnalyze(REINDEX_COUNT);
-      }
-    }
+    long startTime = System.currentTimeMillis();
+    Logging.perf.debug("Beginning to analyze jobqueue table");
+    analyzeTable();
+    Logging.perf.debug("Done analyzing jobqueue table in "+new Long(System.currentTimeMillis()-startTime)+" ms");
   }
 
   /** Uninstall.
@@ -363,7 +320,6 @@ public class JobQueue extends org.apache
     performUpdate(map,"WHERE "+failTimeField+" IS NOT NULL",null,null);
     // Reindex the jobqueue table, since we've probably made lots of bad tuples doing the above operations.
     reindexTable();
-    reindexTracker.doAnalyze(REINDEX_COUNT);
     unconditionallyAnalyzeTables();
   }
 
@@ -457,7 +413,7 @@ public class JobQueue extends org.apache
       quoteSQLString(statusToString(STATUS_COMPLETE))+")",list,null);
 
     // Not accurate, but best we can do without overhead
-    reindexTracker.noteInsert(2);
+    noteModifications(0,2,0);
     // Do an analyze, otherwise our plans are going to be crap right off the bat
     unconditionallyAnalyzeTables();
   }
@@ -489,7 +445,7 @@ public class JobQueue extends org.apache
     map.put(failCountField,null);
     performUpdate(map,"WHERE "+jobIDField+"=? AND "+statusField+"="+
       quoteSQLString(statusToString(STATUS_COMPLETE)),list,null);
-    reindexTracker.noteInsert();
+    noteModifications(0,1,0);
     // Do an analyze, otherwise our plans are going to be crap right off the bat
     unconditionallyAnalyzeTables();
   }
@@ -514,7 +470,7 @@ public class JobQueue extends org.apache
       i++;
     }
     doDeletes(list,sb.toString());
-    reindexTracker.noteInsert(identifiers.length);
+    noteModifications(0,0,identifiers.length);
   }
 
   /** Check if there are any outstanding active documents for a job */
@@ -543,7 +499,7 @@ public class JobQueue extends org.apache
     // Clean out prereqevents table first
     prereqEventManager.deleteRows(getTableName()+" t0","t0."+idField,"t0."+jobIDField+"=?",list);
     performDelete("WHERE "+jobIDField+"=?",list,null);
-    reindexTracker.noteInsert();
+    noteModifications(0,0,1);
   }
 
   /** Write out a document priority */
@@ -556,7 +512,7 @@ public class JobQueue extends org.apache
     ArrayList list = new ArrayList();
     list.add(rowID);
     performUpdate(map,"WHERE "+idField+"=?",list,null);
-    reindexTracker.noteInsert();
+    noteModifications(0,1,0);
   }
 
   /** Set the "completed" status for a record.
@@ -625,7 +581,7 @@ public class JobQueue extends org.apache
     HashMap map = new HashMap();
     map.put(statusField,statusToString(newStatus));
     performUpdate(map,"WHERE "+idField+"=?",list,null);
-    reindexTracker.noteInsert();
+    noteModifications(0,1,0);
   }
 
   /** Set the status on a record, including check time and priority.
@@ -654,7 +610,7 @@ public class JobQueue extends org.apache
     // This does not need to set docPriorityField, because we want to preserve whatever
     // priority was in place from before.
     performUpdate(map,"WHERE "+idField+"=?",list,null);
-    reindexTracker.noteInsert();
+    noteModifications(0,1,0);
   }
 
   /** Set the status of a document to "being deleted".
@@ -667,7 +623,7 @@ public class JobQueue extends org.apache
     HashMap map = new HashMap();
     map.put(statusField,statusToString(STATUS_BEINGDELETED));
     performUpdate(map,"WHERE "+idField+"=?",list,null);
-    reindexTracker.noteInsert();
+    noteModifications(0,1,0);
   }
 
   /** Set the status of a document to be "no longer deleting" */
@@ -714,7 +670,7 @@ public class JobQueue extends org.apache
     }
     if (j > 0)
       doDeletes(list,sb.toString());
-    reindexTracker.noteInsert(ids.length);
+    noteModifications(0,0,ids.length);
   }
 
   /** Do a batch of deletes.
@@ -821,7 +777,7 @@ public class JobQueue extends org.apache
     performUpdate(map,"WHERE "+idField+"=?",list,null);
     // Insert prereqevent entries, if any
     prereqEventManager.addRows(recordID,prereqEvents);
-    reindexTracker.noteInsert();
+    noteModifications(0,1,0);
     return rval;
   }
 
@@ -854,7 +810,7 @@ public class JobQueue extends org.apache
     map.put(prioritySetField,new Long(currentTime));
     performInsert(map,null);
     prereqEventManager.addRows(recordID,prereqEvents);
-    tracker.noteInsert();
+    noteModifications(1,0,0);
   }
 
   /** Note the remaining documents that do NOT need to be queued.  These are noted so that the
@@ -934,7 +890,7 @@ public class JobQueue extends org.apache
     }
     if (k > 0)
       updateRemainingDocuments(sb.toString(),list);
-    reindexTracker.noteInsert(docIDHashes.length);
+    noteModifications(0,docIDHashes.length,0);
   }
 
   /** Process the specified set of documents. */
@@ -1138,7 +1094,7 @@ public class JobQueue extends org.apache
     prereqEventManager.deleteRows(recordID);
     performUpdate(map,"WHERE "+idField+"=?",list,null);
     prereqEventManager.addRows(recordID,prereqEvents);
-    reindexTracker.noteInsert();
+    noteModifications(0,1,0);
     return rval;
   }
 
@@ -1163,7 +1119,7 @@ public class JobQueue extends org.apache
     map.put(prioritySetField,new Long(currentTime));
     performInsert(map,null);
     prereqEventManager.addRows(recordID,prereqEvents);
-    tracker.noteInsert();
+    noteModifications(1,0,0);
   }
 
   // Methods to convert status strings to integers and back
@@ -1282,58 +1238,6 @@ public class JobQueue extends org.apache
     return ManifoldCF.hash(documentIdentifier);
   }
 
-  /** Analyze tracker class.
-  */
-  protected static class AnalyzeTracker
-  {
-    // Number of records to insert before we need to analyze again.
-    // After start, we wait 1000 before analyzing the first time.
-    protected long recordCount = 1000L;
-    protected boolean busy = false;
-
-    /** Constructor.
-    */
-    public AnalyzeTracker()
-    {
-
-    }
-
-    /** Note an analyze.
-    */
-    public synchronized void doAnalyze(long repeatCount)
-    {
-      recordCount = repeatCount;
-      busy = false;
-    }
-
-    public synchronized void noteInsert(int count)
-    {
-      if (recordCount >= (long)count)
-        recordCount -= (long)count;
-      else
-        recordCount = 0L;
-    }
-
-    /** Note an insert */
-    public synchronized void noteInsert()
-    {
-      if (recordCount > 0L)
-        recordCount--;
-    }
-
-    /** Prepare to insert/delete a record, and see if analyze is required.
-    */
-    public synchronized boolean checkAnalyze()
-    {
-      if (busy)
-        return false;
-      busy = (recordCount == 0L);
-      return busy;
-    }
-
-
-  }
-
   // This class filters an ordered resultset to return only the duplicates
   protected static class DuplicateFinder implements ILimitChecker
   {

Modified: incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/PrereqEventManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/PrereqEventManager.java?rev=1023365&r1=1023364&r2=1023365&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/PrereqEventManager.java (original)
+++ incubator/lcf/trunk/modules/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/PrereqEventManager.java Sat Oct 16 19:45:45 2010
@@ -35,17 +35,6 @@ public class PrereqEventManager extends 
   public final static String ownerField = "owner";
   public final static String eventNameField = "eventname";
 
-  // Count of events to call analyze for
-  protected static final long ANALYZE_COUNT = 50000L;
-  // Count of events to call reindex for
-  protected static final long REINDEX_COUNT = 250000L;
-
-  /** Counter for kicking off analyze */
-  protected static Tracker tracker = new Tracker(ANALYZE_COUNT/2);
-  /** Counter for kicking off reindex */
-  protected static Tracker reindexTracker = new Tracker(REINDEX_COUNT/2);
-
-
   /** Constructor.
   *@param database is the database handle.
   */
@@ -139,7 +128,7 @@ public class PrereqEventManager extends 
       sb.append(" AND ").append(parentCriteria);
     sb.append(")");
     performDelete(sb.toString(),list,null);
-    reindexTracker.noteEvent();
+    noteModifications(0,0,1);
   }
 
   /** Delete specified rows, as directly specified without a join. */
@@ -147,7 +136,7 @@ public class PrereqEventManager extends 
     throws ManifoldCFException
   {
     performDelete("WHERE "+ownerField+" IN("+ownerQueryPart+")",list,null);
-    reindexTracker.noteEvent();
+    noteModifications(0,0,1);
   }
 
   /** Delete rows pertaining to a single entry */
@@ -157,7 +146,7 @@ public class PrereqEventManager extends 
     ArrayList list = new ArrayList();
     list.add(recordID);
     performDelete(" WHERE "+ownerField+"=?",list,null);
-    reindexTracker.noteEvent();
+    noteModifications(0,0,1);
   }
 
   /** Add rows pertaining to a single entry */
@@ -174,91 +163,9 @@ public class PrereqEventManager extends 
         map.put(eventNameField,eventNames[i++]);
         performInsert(map,null);
       }
-      tracker.noteEvent(i);
+      noteModifications(i,0,0);
     }
   }
 
-  /** Conditionally do analyze operation.
-  */
-  public void conditionallyMaintainTables()
-    throws ManifoldCFException
-  {
-    if (tracker.checkAction())
-    {
-      try
-      {
-        // Do the analyze
-        analyzeTable();
-      }
-      finally
-      {
-        tracker.noteAction(ANALYZE_COUNT);
-      }
-    }
-    if (reindexTracker.checkAction())
-    {
-      try
-      {
-        // Do the reindex
-        reindexTable();
-      }
-      finally
-      {
-        reindexTracker.noteAction(REINDEX_COUNT);
-      }
-    }
-  }
-
-
-  /** Analyze tracker class.
-  */
-  protected static class Tracker
-  {
-    // Number of records to track before kicking off the action.
-    protected long recordCount;
-    protected boolean busy = false;
-
-    /** Constructor.
-    */
-    public Tracker(long initialValue)
-    {
-      recordCount = initialValue;
-    }
-
-    /** Note an analyze.
-    */
-    public synchronized void noteAction(long repeatCount)
-    {
-      recordCount = repeatCount;
-      busy = false;
-    }
-
-    public synchronized void noteEvent(int count)
-    {
-      if (recordCount >= (long)count)
-        recordCount -= (long)count;
-      else
-        recordCount = 0L;
-    }
-
-    /** Note an insert */
-    public synchronized void noteEvent()
-    {
-      if (recordCount > 0L)
-        recordCount--;
-    }
-
-    /** See if action is required.
-    */
-    public synchronized boolean checkAction()
-    {
-      if (busy)
-        return false;
-      busy = (recordCount == 0L);
-      return busy;
-    }
-
-
-  }
 
 }



Mime
View raw message