Mailing-List: contact connectors-commits-help@incubator.apache.org;
 run by ezmlm
Precedence: bulk
Reply-To: connectors-dev@incubator.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1082598 - in /incubator/lcf/trunk: CHANGES.txt
 framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
 framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
Date: Thu, 17 Mar 2011 17:09:11 -0000
To: connectors-commits@incubator.apache.org
From: kwright@apache.org
Message-Id: <20110317170911.A2F5923888CB@eris.apache.org>

Author: kwright
Date: Thu Mar 17 17:09:11 2011
New Revision: 1082598

URL: http://svn.apache.org/viewvc?rev=1082598&view=rev
Log:
Refactor PostgreSQL driver to make it readable (and, at the same time, remove extra analysis invocation that seems to be a duplicate).  Also add analysis and reindexing capabilities to Derby driver, since 10.7.1.1 supports this now.

Modified:
    incubator/lcf/trunk/CHANGES.txt
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java

Modified: incubator/lcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/CHANGES.txt?rev=1082598&r1=1082597&r2=1082598&view=diff
==============================================================================
--- incubator/lcf/trunk/CHANGES.txt (original)
+++ incubator/lcf/trunk/CHANGES.txt Thu Mar 17 17:09:11 2011
@@ -3,6 +3,10 @@ $Id$
 
 ==================  0.2-dev ==================
 
+CONNECTORS-170: Add support to the Derby driver for periodic analysis and
+reindexing, since after 10.7.1.1 it seems this is now supported.
+(Karl Wright)
+
 CONNECTORS-169: Add a method to the database abstraction to
 return the maximum number of OR clauses in a query.  This is to make
 Derby efficient, since it can't seem to use indexes in this situation

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java?rev=1082598&r1=1082597&r2=1082598&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java Thu Mar 17 17:09:11 2011
@@ -35,15 +35,51 @@ public class DBInterfaceDerby extends Da
   
   public final static String databasePathProperty = "org.apache.manifoldcf.derbydatabasepath";
   
+  /** A lock manager handle. */
+  protected ILockManager lockManager;
 
+  // Credentials
   protected String userName;
   protected String password;
   
+  // Database cache key
   protected String cacheKey;
-  // Postgresql serializable transactions are broken in that transactions that occur within them do not in fact work properly.
-  // So, once we enter the serializable realm, STOP any additional transactions from doing anything at all.
+  
+  // Once we enter the serializable realm, STOP any additional transactions from doing anything at all.
   protected int serializableDepth = 0;
 
+  // Internal transaction depth, and flag whether we're in a transaction or not
+  int depthCount = 0;
+  boolean inTransaction = false;
+  
+  // This is where we keep track of tables that we need to analyze on transaction exit
+  protected ArrayList tablesToAnalyze = new ArrayList();
+
+  // Keep track of tables to reindex on transaction exit
+  protected ArrayList tablesToReindex = new ArrayList();
+
+  // This is where we keep temporary table statistics, which accumulate until they reach a threshold, and then are added into shared memory.
+  
+  /** Accumulated reindex statistics.  This map is keyed by the table name, and contains TableStatistics values. */
+  protected static Map currentReindexStatistics = new HashMap();
+  /** Table reindex thresholds, as read from configuration information.  Keyed by table name, contains Integer values. */
+  protected static Map reindexThresholds = new HashMap();
+  
+  /** Accumulated analyze statistics.  This map is keyed by the table name, and contains TableStatistics values. */
+  protected static Map currentAnalyzeStatistics = new HashMap();
+  /** Table analyze thresholds, as read from configuration information.  Keyed by table name, contains Integer values. */
+  protected static Map analyzeThresholds = new HashMap();
+  
+  /** The number of inserts, deletes, etc. before we update the shared area. */
+  protected static final int commitThreshold = 100;
+
+  // Lock and shared datum name prefixes (to be combined with table names)
+  protected static final String statslockReindexPrefix = "statslock-reindex-";
+  protected static final String statsReindexPrefix = "stats-reindex-";
+  protected static final String statslockAnalyzePrefix = "statslock-analyze-";
+  protected static final String statsAnalyzePrefix = "stats-analyze-";
+  
+
   // Override the Derby default lock timeout, and make it wait indefinitely instead.
   static
   {
@@ -67,6 +103,7 @@ public class DBInterfaceDerby extends Da
   {
     super(tc,_url+getFullDatabasePath(databaseName)+";user="+userName+";password="+password,_driver,getFullDatabasePath(databaseName),userName,password);
     cacheKey = CacheKeyFactory.makeDatabaseKey(this.databaseName);
+    lockManager = LockManagerFactory.make(tc);
     this.userName = userName;
     this.password = password;
   }
@@ -496,13 +533,63 @@ public class DBInterfaceDerby extends Da
     performModification("DROP INDEX "+indexName,null,null);
   }
 
+  /** Read a datum, presuming zero if the datum does not exist.
+  */
+  protected int readDatum(String datumName)
+    throws ManifoldCFException
+  {
+    byte[] bytes = lockManager.readData(datumName);
+    if (bytes == null)
+      return 0;
+    return (((int)bytes[0]) & 0xff) + ((((int)bytes[1]) & 0xff) << 8) + ((((int)bytes[2]) & 0xff) << 16) + ((((int)bytes[3]) & 0xff) << 24);
+  }
+
+  /** Write a datum, presuming zero if the datum does not exist.
+  */
+  protected void writeDatum(String datumName, int value)
+    throws ManifoldCFException
+  {
+    byte[] bytes = new byte[4];
+    bytes[0] = (byte)(value & 0xff);
+    bytes[1] = (byte)((value >> 8) & 0xff);
+    bytes[2] = (byte)((value >> 16) & 0xff);
+    bytes[3] = (byte)((value >> 24) & 0xff);
+    
+    lockManager.writeData(datumName,bytes);
+  }
+
   /** Analyze a table.
   *@param tableName is the name of the table to analyze/calculate statistics for.
   */
   public void analyzeTable(String tableName)
     throws ManifoldCFException
   {
-    // Does nothing on Derby
+    String tableStatisticsLock = statslockAnalyzePrefix+tableName;
+    lockManager.enterWriteCriticalSection(tableStatisticsLock);
+    try
+    {
+      TableStatistics ts = (TableStatistics)currentAnalyzeStatistics.get(tableName);
+      // Lock this table's statistics files
+      lockManager.enterWriteLock(tableStatisticsLock);
+      try
+      {
+        String eventDatum = statsAnalyzePrefix+tableName;
+        // Time to reindex this table!
+        analyzeTableInternal(tableName);
+        // Now, clear out the data
+        writeDatum(eventDatum,0);
+        if (ts != null)
+          ts.reset();
+      }
+      finally
+      {
+        lockManager.leaveWriteLock(tableStatisticsLock);
+      }
+    }
+    finally
+    {
+      lockManager.leaveWriteCriticalSection(tableStatisticsLock);
+    }
   }
 
   /** Reindex a table.
@@ -511,7 +598,90 @@ public class DBInterfaceDerby extends Da
   public void reindexTable(String tableName)
     throws ManifoldCFException
   {
-    // Does nothing on Derby
+    String tableStatisticsLock;
+    
+    // Reindexing.
+    tableStatisticsLock = statslockReindexPrefix+tableName;
+    lockManager.enterWriteCriticalSection(tableStatisticsLock);
+    try
+    {
+      TableStatistics ts = (TableStatistics)currentReindexStatistics.get(tableName);
+      // Lock this table's statistics files
+      lockManager.enterWriteLock(tableStatisticsLock);
+      try
+      {
+        String eventDatum = statsReindexPrefix+tableName;
+        // Time to reindex this table!
+        reindexTableInternal(tableName);
+        // Now, clear out the data
+        writeDatum(eventDatum,0);
+        if (ts != null)
+          ts.reset();
+      }
+      finally
+      {
+        lockManager.leaveWriteLock(tableStatisticsLock);
+      }
+    }
+    finally
+    {
+      lockManager.leaveWriteCriticalSection(tableStatisticsLock);
+    }
+  }
+
+  protected void analyzeTableInternal(String tableName)
+    throws ManifoldCFException
+  {
+    if (getTransactionID() == null)
+    {
+      ArrayList list = new ArrayList();
+      list.add("APP");
+      list.add(tableName.toUpperCase());
+      performModification("CALL SYSCS_UTIL.SYSCS_UPDATE_STATISTICS(?,?,null)",list,null);
+    }
+    else
+      tablesToAnalyze.add(tableName);
+  }
+
+  protected void reindexTableInternal(String tableName)
+    throws ManifoldCFException
+  {
+    if (getTransactionID() == null)
+    {
+      long sleepAmt = 0L;
+      while (true)
+      {
+        try
+        {
+          // To reindex, we (a) get all the table's indexes, (b) drop them, (c) recreate them
+          Map x = getTableIndexes(tableName,null,null);
+          Iterator iter = x.keySet().iterator();
+          while (iter.hasNext())
+          {
+            String indexName = (String)iter.next();
+            IndexDescription id = (IndexDescription)x.get(indexName);
+            performRemoveIndex(indexName);
+            performAddIndex(indexName,tableName,id);
+          }
+          break;
+        }
+        catch (ManifoldCFException e)
+        {
+          if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
+          {
+            sleepAmt = getSleepAmt();
+            continue;
+          }
+          throw e;
+        }
+        finally
+        {
+          sleepFor(sleepAmt);
+        }
+      }
+    }
+    else
+      tablesToReindex.add(tableName);
   }
 
   /** Perform a table drop operation.
@@ -974,6 +1144,134 @@ public class DBInterfaceDerby extends Da
     return 1;
   }
 
+  /** Note a number of inserts, modifications, or deletions to a specific table.  This is so we can decide when to do appropriate maintenance.
+  *@param tableName is the name of the table being modified.
+  *@param insertCount is the number of inserts.
+  *@param modifyCount is the number of updates.
+  *@param deleteCount is the number of deletions.
+  */
+  public void noteModifications(String tableName, int insertCount, int modifyCount, int deleteCount)
+    throws ManifoldCFException
+  {
+    String tableStatisticsLock;
+    int eventCount;
+    
+    // Reindexing.
+    // Here we count tuple deletion.  So we want to know the deletecount + modifycount.
+    eventCount = modifyCount + deleteCount;
+    tableStatisticsLock = statslockReindexPrefix+tableName;
+    lockManager.enterWriteCriticalSection(tableStatisticsLock);
+    try
+    {
+      Integer threshold = (Integer)reindexThresholds.get(tableName);
+      int reindexThreshold;
+      if (threshold == null)
+      {
+        // Look for this parameter; if we don't find it, use a default value.
+        reindexThreshold = ManifoldCF.getIntProperty("org.apache.manifold.db.derby.reindex."+tableName,250000);
+        reindexThresholds.put(tableName,new Integer(reindexThreshold));
+      }
+      else
+        reindexThreshold = threshold.intValue();
+      
+      TableStatistics ts = (TableStatistics)currentReindexStatistics.get(tableName);
+      if (ts == null)
+      {
+        ts = new TableStatistics();
+        currentReindexStatistics.put(tableName,ts);
+      }
+      ts.add(eventCount);
+      // Check if we have passed threshold yet for this table, for committing the data to the shared area
+      if (ts.getEventCount() >= commitThreshold)
+      {
+        // Lock this table's statistics files
+        lockManager.enterWriteLock(tableStatisticsLock);
+        try
+        {
+          String eventDatum = statsReindexPrefix+tableName;
+          int oldEventCount = readDatum(eventDatum);
+          oldEventCount += ts.getEventCount();
+          if (oldEventCount >= reindexThreshold)
+          {
+            // Time to reindex this table!
+            reindexTableInternal(tableName);
+            // Now, clear out the data
+            writeDatum(eventDatum,0);
+          }
+          else
+            writeDatum(eventDatum,oldEventCount);
+          ts.reset();
+        }
+        finally
+        {
+          lockManager.leaveWriteLock(tableStatisticsLock);
+        }
+      }
+    }
+    finally
+    {
+      lockManager.leaveWriteCriticalSection(tableStatisticsLock);
+    }
+    
+      // Analysis.
+    // Here we count tuple addition.
+    eventCount = modifyCount + insertCount;
+    tableStatisticsLock = statslockAnalyzePrefix+tableName;
+    lockManager.enterWriteCriticalSection(tableStatisticsLock);
+    try
+    {
+      Integer threshold = (Integer)analyzeThresholds.get(tableName);
+      int analyzeThreshold;
+      if (threshold == null)
+      {
+        // Look for this parameter; if we don't find it, use a default value.
+        analyzeThreshold = ManifoldCF.getIntProperty("org.apache.manifold.db.derby.analyze."+tableName,5000);
+        analyzeThresholds.put(tableName,new Integer(analyzeThreshold));
+      }
+      else
+        analyzeThreshold = threshold.intValue();
+      
+      TableStatistics ts = (TableStatistics)currentAnalyzeStatistics.get(tableName);
+      if (ts == null)
+      {
+        ts = new TableStatistics();
+        currentAnalyzeStatistics.put(tableName,ts);
+      }
+      ts.add(eventCount);
+      // Check if we have passed threshold yet for this table, for committing the data to the shared area
+      if (ts.getEventCount() >= commitThreshold)
+      {
+        // Lock this table's statistics files
+        lockManager.enterWriteLock(tableStatisticsLock);
+        try
+        {
+          String eventDatum = statsAnalyzePrefix+tableName;
+          int oldEventCount = readDatum(eventDatum);
+          oldEventCount += ts.getEventCount();
+          if (oldEventCount >= analyzeThreshold)
+          {
+            // Time to reindex this table!
+            analyzeTableInternal(tableName);
+            // Now, clear out the data
+            writeDatum(eventDatum,0);
+          }
+          else
+            writeDatum(eventDatum,oldEventCount);
+          ts.reset();
+        }
+        finally
+        {
+          lockManager.leaveWriteLock(tableStatisticsLock);
+        }
+      }
+    }
+    finally
+    {
+      lockManager.leaveWriteCriticalSection(tableStatisticsLock);
+    }
+
+  }
+  
 
   /** Begin a database transaction.  This method call MUST be paired with an endTransaction() call,
   * or database handles will be lost.  If the transaction should be rolled back, then signalRollback() should
@@ -1078,11 +1376,24 @@ public class DBInterfaceDerby extends Da
     }
 
     super.endTransaction();
+    if (getTransactionID() == null)
+    {
+      int i = 0;
+      while (i < tablesToAnalyze.size())
+      {
+        analyzeTableInternal((String)tablesToAnalyze.get(i++));
+      }
+      tablesToAnalyze.clear();
+      i = 0;
+      while (i < tablesToReindex.size())
+      {
+        reindexTableInternal((String)tablesToReindex.get(i++));
+      }
+      tablesToReindex.clear();
+    }
+
   }
 
-  int depthCount = 0;
-  boolean inTransaction = false;
-  
   /** Abstract method to start a transaction */
   protected void startATransaction()
     throws ManifoldCFException
@@ -1164,6 +1475,7 @@ public class DBInterfaceDerby extends Da
     return rawColumnName.toLowerCase();
   }
 
+
   // Functions that correspond to user-defined functions in Derby
   
   /** Method to compare a value using a case-insensitive regular expression.
@@ -1252,5 +1564,31 @@ public class DBInterfaceDerby extends Da
     }
   }
 
+  /** Table accumulation records.
+  */
+  protected static class TableStatistics
+  {
+    protected int eventCount = 0;
+    
+    public TableStatistics()
+    {
+    }
+    
+    public void reset()
+    {
+      eventCount = 0;
+    }
+    
+    public void add(int eventCount)
+    {
+      this.eventCount += eventCount;
+    }
+    
+    public int getEventCount()
+    {
+      return eventCount;
+    }
+  }
+
 }
 

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java?rev=1082598&r1=1082597&r2=1082598&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java (original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java Thu Mar 17 17:09:11 2011
@@ -40,7 +40,9 @@ public class DBInterfacePostgreSQL exten
   /** A lock manager handle. */
   protected ILockManager lockManager;
   
+  // Database cache key
   protected String cacheKey;
+	
   // Postgresql serializable transactions are broken in that transactions that occur within them do not in fact work properly.
   // So, once we enter the serializable realm, STOP any additional transactions from doing anything at all.
   protected int serializableDepth = 0;
@@ -51,6 +53,28 @@ public class DBInterfacePostgreSQL exten
   // Keep track of tables to reindex on transaction exit
   protected ArrayList tablesToReindex = new ArrayList();
 
+  // This is where we keep temporary table statistics, which accumulate until they reach a threshold, and then are added into shared memory.
+  
+  /** Accumulated reindex statistics.  This map is keyed by the table name, and contains TableStatistics values. */
+  protected static Map currentReindexStatistics = new HashMap();
+  /** Table reindex thresholds, as read from configuration information.  Keyed by table name, contains Integer values. */
+  protected static Map reindexThresholds = new HashMap();
+  
+  /** Accumulated analyze statistics.  This map is keyed by the table name, and contains TableStatistics values. */
+  protected static Map currentAnalyzeStatistics = new HashMap();
+  /** Table analyze thresholds, as read from configuration information.  Keyed by table name, contains Integer values. */
+  protected static Map analyzeThresholds = new HashMap();
+  
+  /** The number of inserts, deletes, etc. before we update the shared area. */
+  protected static final int commitThreshold = 100;
+
+  // Lock and shared datum name prefixes (to be combined with table names)
+  protected static final String statslockReindexPrefix = "statslock-reindex-";
+  protected static final String statsReindexPrefix = "stats-reindex-";
+  protected static final String statslockAnalyzePrefix = "statslock-analyze-";
+  protected static final String statsAnalyzePrefix = "stats-analyze-";
+  
+
   public DBInterfacePostgreSQL(IThreadContext tc, String databaseName, String userName, String password)
     throws ManifoldCFException
   {
@@ -477,46 +501,6 @@ public class DBInterfacePostgreSQL exten
     performModification("DROP INDEX "+indexName,null,null);
   }
 
-  protected void analyzeTableInternal(String tableName)
-    throws ManifoldCFException
-  {
-    if (getTransactionID() == null)
-      performModification("ANALYZE "+tableName,null,null);
-    else
-      tablesToAnalyze.add(tableName);
-  }
-
-  protected void reindexTableInternal(String tableName)
-    throws ManifoldCFException
-  {
-    if (getTransactionID() == null)
-    {
-      long sleepAmt = 0L;
-      while (true)
-      {
-        try
-        {
-          performModification("REINDEX TABLE "+tableName,null,null);
-          break;
-        }
-        catch (ManifoldCFException e)
-        {
-          if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
-          {
-            sleepAmt = getSleepAmt();
-            continue;
-          }
-          throw e;
-        }
-        finally
-        {
-          sleepFor(sleepAmt);
-        }
-      }
-    }
-    else
-      tablesToReindex.add(tableName);
-  }
 
   /** Perform a table drop operation.
   *@param tableName is the name of the table to drop.
@@ -1141,21 +1125,6 @@ public class DBInterfacePostgreSQL exten
     Logging.db.warn("");
   }
 
-  // This is where we keep temporary table statistics, which accumulate until they reach a threshold, and then are added into shared memory.
-  
-  /** Accumulated reindex statistics.  This map is keyed by the table name, and contains TableStatistics values. */
-  protected static Map currentReindexStatistics = new HashMap();
-  /** Table reindex thresholds, as read from configuration information.  Keyed by table name, contains Integer values. */
-  protected static Map reindexThresholds = new HashMap();
-  
-  /** Accumulated analyze statistics.  This map is keyed by the table name, and contains TableStatistics values. */
-  protected static Map currentAnalyzeStatistics = new HashMap();
-  /** Table analyze thresholds, as read from configuration information.  Keyed by table name, contains Integer values. */
-  protected static Map analyzeThresholds = new HashMap();
-  
-  /** The number of inserts, deletes, etc. before we update the shared area.
-  */
-  protected static final int commitThreshold = 100;
   
   /** Read a datum, presuming zero if the datum does not exist.
   */
@@ -1182,12 +1151,6 @@ public class DBInterfacePostgreSQL exten
     lockManager.writeData(datumName,bytes);
   }
 
-  // Lock and shared datum name prefixes (to be combined with table names)
-  protected static final String statslockReindexPrefix = "statslock-reindex-";
-  protected static final String statsReindexPrefix = "stats-reindex-";
-  protected static final String statslockAnalyzePrefix = "statslock-analyze-";
-  protected static final String statsAnalyzePrefix = "stats-analyze-";
-  
   /** Analyze a table.
   *@param tableName is the name of the table to analyze/calculate statistics for.
   */
@@ -1220,8 +1183,6 @@ public class DBInterfacePostgreSQL exten
     {
       lockManager.leaveWriteCriticalSection(tableStatisticsLock);
     }
-
-    analyzeTableInternal(tableName);
   }
 
   /** Reindex a table.
@@ -1261,6 +1222,47 @@ public class DBInterfacePostgreSQL exten
     }
   }
 
+  protected void analyzeTableInternal(String tableName)
+    throws ManifoldCFException
+  {
+    if (getTransactionID() == null)
+      performModification("ANALYZE "+tableName,null,null);
+    else
+      tablesToAnalyze.add(tableName);
+  }
+
+  protected void reindexTableInternal(String tableName)
+    throws ManifoldCFException
+  {
+    if (getTransactionID() == null)
+    {
+      long sleepAmt = 0L;
+      while (true)
+      {
+        try
+        {
+          performModification("REINDEX TABLE "+tableName,null,null);
+          break;
+        }
+        catch (ManifoldCFException e)
+        {
+          if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
+          {
+            sleepAmt = getSleepAmt();
+            continue;
+          }
+          throw e;
+        }
+        finally
+        {
+          sleepFor(sleepAmt);
+        }
+      }
+    }
+    else
+      tablesToReindex.add(tableName);
+  }
+
   /** Note a number of inserts, modifications, or deletions to a specific table.  This is so we can decide when to do appropriate maintenance.
   *@param tableName is the name of the table being modified.
   *@param insertCount is the number of inserts.