incubator-connectors-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1022518 - in /incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core: database/DBInterfacePostgreSQL.java database/Database.java interfaces/IDBInterface.java system/ManifoldCF.java
Date Thu, 14 Oct 2010 13:19:49 GMT
Author: kwright
Date: Thu Oct 14 13:19:49 2010
New Revision: 1022518

URL: http://svn.apache.org/viewvc?rev=1022518&view=rev
Log:
Add infrastructure support for database table maintenance, so that it is handled under the
abstraction barrier.  This is the first phase of the CONNECTORS-117 work.

Modified:
    incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
    incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
    incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
    incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java

Modified: incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java?rev=1022518&r1=1022517&r2=1022518&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
(original)
+++ incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
Thu Oct 14 13:19:49 2010
@@ -19,6 +19,7 @@
 package org.apache.manifoldcf.core.database;
 
 import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.core.system.ManifoldCF;
 import org.apache.manifoldcf.core.system.Logging;
 import java.util.*;
 
@@ -29,6 +30,9 @@ public class DBInterfacePostgreSQL exten
   private static final String _url = "jdbc:postgresql://localhost/";
   private static final String _driver = "org.postgresql.Driver";
 
+  /** A lock manager handle. */
+  protected ILockManager lockManager;
+  
   protected String cacheKey;
   // Postgresql serializable transactions are broken in that transactions that occur within
them do not in fact work properly.
   // So, once we enter the serializable realm, STOP any additional transactions from doing
anything at all.
@@ -45,6 +49,7 @@ public class DBInterfacePostgreSQL exten
   {
     super(tc,_url+databaseName,_driver,databaseName,userName,password);
     cacheKey = CacheKeyFactory.makeDatabaseKey(this.databaseName);
+    lockManager = LockManagerFactory.make(tc);
   }
 
   /** Initialize.  This method is called once per JVM instance, in order to set up
@@ -1094,5 +1099,200 @@ public class DBInterfacePostgreSQL exten
     Logging.db.warn("");
   }
 
+  // This is where we keep temporary table statistics, which accumulate until they reach
a threshold, and then are added into shared memory.
+  
+  /** Accumulated reindex statistics.  This map is keyed by the table name, and contains
TableStatistics values. */
+  protected static Map currentReindexStatistics = new HashMap();
+  /** Table reindex thresholds, as read from configuration information.  Keyed by table name,
contains Integer values. */
+  protected static Map reindexThresholds = new HashMap();
+  
+  /** Accumulated analyze statistics.  This map is keyed by the table name, and contains
TableStatistics values. */
+  protected static Map currentAnalyzeStatistics = new HashMap();
+  /** Table analyze thresholds, as read from configuration information.  Keyed by table name,
contains Integer values. */
+  protected static Map analyzeThresholds = new HashMap();
+  
+  /** The number of inserts, deletes, etc. before we update the shared area.
+  */
+  protected static final int commitThreshold = 100;
+  
+  /** Read a datum, presuming zero if the datum does not exist.
+  */
+  protected int readDatum(String datumName)
+    throws ManifoldCFException
+  {
+    byte[] bytes = lockManager.readData(datumName);
+    if (bytes == null)
+      return 0;
+    return (((int)bytes[0]) & 0xff) + ((((int)bytes[1]) & 0xff) << 8) + ((((int)bytes[2])
& 0xff) << 16) + ((((int)bytes[3]) & 0xff) << 24);
+  }
+
+  /** Write a datum, presuming zero if the datum does not exist.
+  */
+  protected void writeDatum(String datumName, int value)
+    throws ManifoldCFException
+  {
+    byte[] bytes = new byte[4];
+    bytes[0] = (byte)(value & 0xff);
+    bytes[1] = (byte)((value >> 8) & 0xff);
+    bytes[2] = (byte)((value >> 16) & 0xff);
+    bytes[3] = (byte)((value >> 24) & 0xff);
+    
+    lockManager.writeData(datumName,bytes);
+  }
+
+  /** Note a number of inserts, modifications, or deletions to a specific table.  This is
so we can decide when to do appropriate maintenance.
+  *@param tableName is the name of the table being modified.
+  *@param insertCount is the number of inserts.
+  *@param modifyCount is the number of updates.
+  *@param deleteCount is the number of deletions.
+  */
+  public void noteModifications(String tableName, int insertCount, int modifyCount, int deleteCount)
+    throws ManifoldCFException
+  {
+    String tableStatisticsLock;
+    int eventCount;
+    
+    // Reindexing.
+    // Here we count tuple deletion.  So we want to know the deletecount + modifycount.
+    eventCount = modifyCount + deleteCount;
+    tableStatisticsLock = "statslock-reindex-"+tableName;
+    lockManager.enterWriteCriticalSection(tableStatisticsLock);
+    try
+    {
+      Integer threshold = (Integer)reindexThresholds.get(tableName);
+      int reindexThreshold;
+      if (threshold == null)
+      {
+        // Look for this parameter; if we don't find it, use a default value.
+        reindexThreshold = ManifoldCF.getIntProperty("org.apache.manifold.db.postgres.reindex."+tableName,100000);
+        reindexThresholds.put(tableName,new Integer(reindexThreshold));
+      }
+      else
+        reindexThreshold = threshold.intValue();
+      
+      TableStatistics ts = (TableStatistics)currentReindexStatistics.get(tableName);
+      if (ts == null)
+      {
+        ts = new TableStatistics();
+        currentReindexStatistics.put(tableName,ts);
+      }
+      ts.add(eventCount);
+      // Check if we have passed threshold yet for this table, for committing the data to
the shared area
+      if (ts.getEventCount() >= commitThreshold)
+      {
+        // Lock this table's statistics files
+        lockManager.enterWriteLock(tableStatisticsLock);
+        try
+        {
+          String eventDatum = "stats-reindex-"+tableName;
+          int oldEventCount = readDatum(eventDatum);
+          oldEventCount += ts.getEventCount();
+          if (oldEventCount >= reindexThreshold)
+          {
+            // Time to reindex this table!
+            reindexTable(tableName);
+            // Now, clear out the data
+            writeDatum(eventDatum,0);
+          }
+          else
+            writeDatum(eventDatum,oldEventCount);
+          ts.reset();
+        }
+        finally
+        {
+          lockManager.leaveWriteLock(tableStatisticsLock);
+        }
+      }
+    }
+    finally
+    {
+      lockManager.leaveWriteCriticalSection(tableStatisticsLock);
+    }
+    
+      // Analysis.
+    // Here we count tuple addition.
+    eventCount = modifyCount + insertCount;
+    tableStatisticsLock = "statslock-analyze-"+tableName;
+    lockManager.enterWriteCriticalSection(tableStatisticsLock);
+    try
+    {
+      Integer threshold = (Integer)analyzeThresholds.get(tableName);
+      int analyzeThreshold;
+      if (threshold == null)
+      {
+        // Look for this parameter; if we don't find it, use a default value.
+        analyzeThreshold = ManifoldCF.getIntProperty("org.apache.manifold.db.postgres.analyze."+tableName,5000);
+        analyzeThresholds.put(tableName,new Integer(analyzeThreshold));
+      }
+      else
+        analyzeThreshold = threshold.intValue();
+      
+      TableStatistics ts = (TableStatistics)currentAnalyzeStatistics.get(tableName);
+      if (ts == null)
+      {
+        ts = new TableStatistics();
+        currentAnalyzeStatistics.put(tableName,ts);
+      }
+      ts.add(eventCount);
+      // Check if we have passed threshold yet for this table, for committing the data to
the shared area
+      if (ts.getEventCount() >= commitThreshold)
+      {
+        // Lock this table's statistics files
+        lockManager.enterWriteLock(tableStatisticsLock);
+        try
+        {
+          String eventDatum = "stats-analyze-"+tableName;
+          int oldEventCount = readDatum(eventDatum);
+          oldEventCount += ts.getEventCount();
+          if (oldEventCount >= analyzeThreshold)
+          {
+            // Time to reindex this table!
+            analyzeTable(tableName);
+            // Now, clear out the data
+            writeDatum(eventDatum,0);
+          }
+          else
+            writeDatum(eventDatum,oldEventCount);
+          ts.reset();
+        }
+        finally
+        {
+          lockManager.leaveWriteLock(tableStatisticsLock);
+        }
+      }
+    }
+    finally
+    {
+      lockManager.leaveWriteCriticalSection(tableStatisticsLock);
+    }
+
+  }
+  
+  /** Table accumulation records.
+  */
+  protected static class TableStatistics
+  {
+    protected int eventCount = 0;
+    
+    public TableStatistics()
+    {
+    }
+    
+    public void reset()
+    {
+      eventCount = 0;
+    }
+    
+    public void add(int eventCount)
+    {
+      this.eventCount += eventCount;
+    }
+    
+    public int getEventCount()
+    {
+      return eventCount;
+    }
+  }
+  
 }
 

Modified: incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java?rev=1022518&r1=1022517&r2=1022518&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
(original)
+++ incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
Thu Oct 14 13:19:49 2010
@@ -343,6 +343,17 @@ public class Database
 
   }
 
+  /** Note a number of inserts, modifications, or deletions to a specific table.  This is
so we can decide when to do appropriate maintenance.
+  *@param tableName is the name of the table being modified.
+  *@param insertCount is the number of inserts.
+  *@param modifyCount is the number of updates.
+  *@param deleteCount is the number of deletions.
+  */
+  public void noteModifications(String tableName, int insertCount, int modifyCount, int deleteCount)
+    throws ManifoldCFException
+  {
+  }
+
   /** Thread used to execute queries.  An instance of this thread is spun up every time a
query is executed.  This is necessary because JDBC does not
   * guarantee interruptability, and the Postgresql JDBC driver unfortunately eats all thread
interrupts.  So, we fire up a thread to do each interaction with
   * the database server, thus insuring that the owning thread remains interruptable and will
therefore not block shutdown.

Modified: incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java?rev=1022518&r1=1022517&r2=1022518&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
(original)
+++ incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
Thu Oct 14 13:19:49 2010
@@ -355,5 +355,14 @@ public interface IDBInterface
   public void endTransaction()
     throws ManifoldCFException;
 
+  /** Note a number of inserts, modifications, or deletions to a specific table.  This is
so we can decide when to do appropriate maintenance.
+  *@param tableName is the name of the table being modified.
+  *@param insertCount is the number of inserts.
+  *@param modifyCount is the number of updates.
+  *@param deleteCount is the number of deletions.
+  */
+  public void noteModifications(String tableName, int insertCount, int modifyCount, int deleteCount)
+    throws ManifoldCFException;
+
 }
 

Modified: incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java?rev=1022518&r1=1022517&r2=1022518&view=diff
==============================================================================
--- incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
(original)
+++ incubator/lcf/trunk/modules/framework/core/src/main/java/org/apache/manifoldcf/core/system/ManifoldCF.java
Thu Oct 14 13:19:49 2010
@@ -303,6 +303,24 @@ public class ManifoldCF
     return resolvePath(value);
   }
   
+  /** Read an integer propert, either from the system properties, or from the local configuration
file.
+  */
+  public static int getIntProperty(String s, int defaultValue)
+    throws ManifoldCFException
+  {
+    String value = getProperty(s);
+    if (value == null)
+      return defaultValue;
+    try
+    {
+      return Integer.parseInt(value);
+    }
+    catch (NumberFormatException e)
+    {
+      throw new ManifoldCFException("Illegal property value for integer property '"+s+"':
'"+value+"': "+e.getMessage(),e,ManifoldCFException.SETUP_ERROR);
+    }
+  }
+  
   /** Attempt to make sure a path is a folder
   * @param path
   */



Mime
View raw message