manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1150502 - in /incubator/lcf/trunk: ./ framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/ framework/core/src/main/java/org/apache/manifoldcf/core/database/ framework/pull-agent/src/main/java/org/apache/manifoldcf...
Date Sun, 24 Jul 2011 22:19:53 GMT
Author: kwright
Date: Sun Jul 24 22:19:52 2011
New Revision: 1150502

URL: http://svn.apache.org/viewvc?rev=1150502&view=rev
Log:
Fix for CONNECTORS-225.  Put a deadlock wait and retry.

Modified:
    incubator/lcf/trunk/CHANGES.txt
    incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java

Modified: incubator/lcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/CHANGES.txt?rev=1150502&r1=1150501&r2=1150502&view=diff
==============================================================================
--- incubator/lcf/trunk/CHANGES.txt (original)
+++ incubator/lcf/trunk/CHANGES.txt Sun Jul 24 22:19:52 2011
@@ -3,6 +3,10 @@ $Id$
 
 ======================= 0.3-dev =========================
 
+CONNECTORS-225: Derby throws deadlock exceptions when indexing
+rapidly.
+(Karl Wright)
+
 CONNECTORS-223: Move test classes to be compatible with maven
 conventions.
 (Karl Wright)

Modified: incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java?rev=1150502&r1=1150501&r2=1150502&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
(original)
+++ incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
Sun Jul 24 22:19:52 2011
@@ -78,7 +78,6 @@ public class IncrementalIngester extends
   // Output connection manager
   protected IOutputConnectionManager connectionManager;
 
-
   /** Constructor.
   */
   public IncrementalIngester(IThreadContext threadContext, IDBInterface database)
@@ -1362,47 +1361,62 @@ public class IncrementalIngester extends
       else
         map.put(authorityNameField,"");
 
-      beginTransaction();
-      try
+      // Transaction abort due to deadlock should be retried here.
+      while (true)
       {
-        // Look for existing row.
-        ArrayList list = new ArrayList();
-        list.add(docKey);
-        list.add(outputConnectionName);
-        IResultSet set = performQuery("SELECT "+idField+","+changeCountField+" FROM "+getTableName()+"
WHERE "+
-          docKeyField+"=? AND "+outputConnNameField+"=? FOR UPDATE",list,null,null);
-        IResultRow row = null;
-        if (set.getRowCount() > 0)
-          row = set.getRow(0);
+        long sleepAmt = 0L;
 
-        if (row != null)
+        beginTransaction();
+        try
         {
-          // Update the record
-          list.clear();
-          list.add(row.getValue(idField));
-          long changeCount = ((Long)row.getValue(changeCountField)).longValue();
-          changeCount++;
-          map.put(changeCountField,new Long(changeCount));
-          performUpdate(map,"WHERE "+idField+"=?",list,null);
-          // Update successful!
-          return;
+          // Look for existing row.
+          ArrayList list = new ArrayList();
+          list.add(docKey);
+          list.add(outputConnectionName);
+          IResultSet set = performQuery("SELECT "+idField+","+changeCountField+" FROM "+getTableName()+"
WHERE "+
+            docKeyField+"=? AND "+outputConnNameField+"=? FOR UPDATE",list,null,null);
+          IResultRow row = null;
+          if (set.getRowCount() > 0)
+            row = set.getRow(0);
+
+          if (row != null)
+          {
+            // Update the record
+            list.clear();
+            list.add(row.getValue(idField));
+            long changeCount = ((Long)row.getValue(changeCountField)).longValue();
+            changeCount++;
+            map.put(changeCountField,new Long(changeCount));
+            performUpdate(map,"WHERE "+idField+"=?",list,null);
+            // Update successful!
+            return;
+          }
+
+          // Update failed to find a matching record, so cycle back to retry the insert
         }
+        catch (ManifoldCFException e)
+        {
+          signalRollback();
+          if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
+          {
+            if (Logging.perf.isDebugEnabled())
+              Logging.perf.debug("Aborted transaction noting ingestion: "+e.getMessage());
+            sleepAmt = getSleepAmt();
+            continue;
+          }
 
-        // Update failed to find a matching record, so cycle back to retry the insert
-      }
-      catch (ManifoldCFException e)
-      {
-        signalRollback();
-        throw e;
-      }
-      catch (Error e)
-      {
-        signalRollback();
-        throw e;
-      }
-      finally
-      {
-        endTransaction();
+          throw e;
+        }
+        catch (Error e)
+        {
+          signalRollback();
+          throw e;
+        }
+        finally
+        {
+          endTransaction();
+          sleepFor(sleepAmt);
+        }
       }
     }
   }

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java?rev=1150502&r1=1150501&r2=1150502&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
Sun Jul 24 22:19:52 2011
@@ -274,6 +274,19 @@ public class BaseTable
     dbInterface.endTransaction();
   }
 
+  /** Get a random amount to sleep for (to resolve a deadlock) */
+  protected long getSleepAmt()
+  {
+    return dbInterface.getSleepAmt();
+  }
+  
+  /** Sleep for a specified amount, to resolve a deadlock */
+  protected void sleepFor(long amt)
+    throws ManifoldCFException
+  {
+    dbInterface.sleepFor(amt);
+  }
+  
   /** Note a number of inserts, modifications, or deletions to a specific table.  This is
so we can decide when to do appropriate maintenance.
   *@param tableName is the name of the table being modified.
   *@param insertCount is the number of inserts.

Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1150502&r1=1150501&r2=1150502&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
(original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Sun Jul 24 22:19:52 2011
@@ -4230,24 +4230,13 @@ public class JobManager implements IJobM
   */
   protected long getRandomAmount()
   {
-    // Amount should be between .5 and 1 minute, approx, to give things time to unwind
-    return (long)(random.nextDouble() * 60000.0 + 500.0);
+    return database.getSleepAmt();
   }
 
   protected void sleepFor(long amt)
     throws ManifoldCFException
   {
-    if (amt == 0L)
-      return;
-
-    try
-    {
-      ManifoldCF.sleep(amt);
-    }
-    catch (InterruptedException e)
-    {
-      throw new ManifoldCFException("Interrupted",e,ManifoldCFException.INTERRUPTED);
-    }
+    database.sleepFor(amt);
   }
 
   /** Retrieve specific parent data for a given document.



Mime
View raw message