incubator-connectors-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1221745 - in /incubator/lcf/trunk/framework: agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/ agents/src/main/java/org/apache/manifoldcf/agents/outputconnmgr/ core/src/main/java/org/apache/manifoldcf/core/database/ core/...
Date Wed, 21 Dec 2011 14:42:00 GMT
Author: kwright
Date: Wed Dec 21 14:41:59 2011
New Revision: 1221745

URL: http://svn.apache.org/viewvc?rev=1221745&view=rev
Log:
Get rid of 'lock table' abstraction, since it is not necessary anymore and causes nothing
but trouble with MySQL.

Modified:
    incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java
    incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnmgr/OutputConnectorManager.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authconnmgr/AuthorityConnectorManager.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connmgr/ConnectorManager.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java

Modified: incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java
(original)
+++ incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnection/OutputConnectionManager.java
Wed Dec 21 14:41:59 2011
@@ -290,7 +290,7 @@ public class OutputConnectionManager ext
           beginTransaction();
           try
           {
-            performLock();
+            //performLock();
             // Notify of a change to the configuration
             ManifoldCF.noteConfigurationChange();
             boolean isNew = object.getIsNew();

Modified: incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnmgr/OutputConnectorManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnmgr/OutputConnectorManager.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnmgr/OutputConnectorManager.java
(original)
+++ incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/outputconnmgr/OutputConnectorManager.java
Wed Dec 21 14:41:59 2011
@@ -158,7 +158,7 @@ public class OutputConnectorManager exte
     beginTransaction();
     try
     {
-      performLock();
+      //performLock();
       // See if already there.
       ArrayList params = new ArrayList();
       params.add(className);

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/BaseTable.java
Wed Dec 21 14:41:59 2011
@@ -57,14 +57,6 @@ public class BaseTable
     return dbInterface.getTransactionID();
   }
 
-  /** Perform a table lock operation.
-  */
-  protected void performLock()
-    throws ManifoldCFException
-  {
-    dbInterface.performLock(tableName);
-  }
-
   /** Perform an insert operation.
   *@param invalidateKeys are the cache keys that should be
   * invalidated.

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceDerby.java
Wed Dec 21 14:41:59 2011
@@ -171,15 +171,6 @@ public class DBInterfaceDerby extends Da
     return cacheKey;
   }
 
-  /** Perform a table lock operation.
-  *@param tableName is the name of the table.
-  */
-  public void performLock(String tableName)
-    throws ManifoldCFException
-  {
-    performModification("LOCK TABLE "+tableName+" IN EXCLUSIVE MODE",null,null);
-  }
-
   /** Perform an insert operation.
   *@param tableName is the name of the table.
   *@param invalidateKeys are the cache keys that should be

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceHSQLDB.java
Wed Dec 21 14:41:59 2011
@@ -192,15 +192,6 @@ public class DBInterfaceHSQLDB extends D
     return cacheKey;
   }
 
-  /** Perform a table lock operation.
-  *@param tableName is the name of the table.
-  */
-  public void performLock(String tableName)
-    throws ManifoldCFException
-  {
-    performModification("LOCK TABLE "+tableName+" WRITE",null,null);
-  }
-
   /** Perform an insert operation.
   *@param tableName is the name of the table.
   *@param invalidateKeys are the cache keys that should be

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfaceMySQL.java
Wed Dec 21 14:41:59 2011
@@ -74,15 +74,6 @@ public class DBInterfaceMySQL extends Da
     return cacheKey;
   }
 
-  /** Perform a table lock operation.
-  *@param tableName is the name of the table.
-  */
-  public void performLock(String tableName)
-    throws ManifoldCFException
-  {
-    //performModification("LOCK TABLE "+tableName+" IN EXCLUSIVE MODE",null,null);
-  }
-
   /** Perform an insert operation.
   *@param tableName is the name of the table.
   *@param invalidateKeys are the cache keys that should be

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
Wed Dec 21 14:41:59 2011
@@ -131,15 +131,6 @@ public class DBInterfacePostgreSQL exten
     return cacheKey;
   }
 
-  /** Perform a table lock operation.
-  *@param tableName is the name of the table.
-  */
-  public void performLock(String tableName)
-    throws ManifoldCFException
-  {
-    performModification("LOCK TABLE "+tableName+" IN EXCLUSIVE MODE",null,null);
-  }
-
   /** Perform an insert operation.
   *@param tableName is the name of the table.
   *@param invalidateKeys are the cache keys that should be

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/IDBInterface.java
Wed Dec 21 14:41:59 2011
@@ -66,12 +66,6 @@ public interface IDBInterface
   */
   public String getDatabaseCacheKey();
 
-  /** Perform a table lock operation.
-  *@param tableName is the name of the table.
-  */
-  public void performLock(String tableName)
-    throws ManifoldCFException;
-
   /** Perform an insert operation.
   *@param tableName is the name of the table.
   *@param invalidateKeys are the cache keys that should be

Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authconnmgr/AuthorityConnectorManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authconnmgr/AuthorityConnectorManager.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authconnmgr/AuthorityConnectorManager.java
(original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authconnmgr/AuthorityConnectorManager.java
Wed Dec 21 14:41:59 2011
@@ -158,7 +158,7 @@ public class AuthorityConnectorManager e
     beginTransaction();
     try
     {
-      performLock();
+      //performLock();
       // See if already there.
       ArrayList params = new ArrayList();
       params.add(className);

Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java
(original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/authorities/authority/AuthorityConnectionManager.java
Wed Dec 21 14:41:59 2011
@@ -267,7 +267,7 @@ public class AuthorityConnectionManager 
           beginTransaction();
           try
           {
-            performLock();
+            //performLock();
             ManifoldCF.noteConfigurationChange();
             boolean isNew = object.getIsNew();
             // See whether the instance exists

Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connmgr/ConnectorManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connmgr/ConnectorManager.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connmgr/ConnectorManager.java
(original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/connmgr/ConnectorManager.java
Wed Dec 21 14:41:59 2011
@@ -154,7 +154,7 @@ public class ConnectorManager extends or
     beginTransaction();
     try
     {
-      performLock();
+      //performLock();
       // See if already there.
       ArrayList params = new ArrayList();
       params.add(className);

Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
(original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/Jobs.java
Wed Dec 21 14:41:59 2011
@@ -672,111 +672,130 @@ public class Jobs extends org.apache.man
     ssb.add(getJobIDKey(id));
     StringSet invKeys = new StringSet(ssb);
 
-    ICacheHandle ch = cacheManager.enterCache(null,invKeys,getTransactionID());
-    try
+    while (true)
     {
-      beginTransaction();
+      long sleepAmt = 0L;
       try
       {
-        performLock();
-        // See whether the instance exists
-        ArrayList params = new ArrayList();
-        String query = buildConjunctionClause(params,new ClauseDescription[]{
-          new UnitaryClause(idField,id)});
-        IResultSet set = performQuery("SELECT * FROM "+getTableName()+" WHERE "+
-          query+" FOR UPDATE",params,null,null);
-        HashMap values = new HashMap();
-        values.put(descriptionField,jobDescription.getDescription());
-        values.put(outputNameField,jobDescription.getOutputConnectionName());
-        values.put(connectionNameField,jobDescription.getConnectionName());
-        String newOutputXML = jobDescription.getOutputSpecification().toXML();
-        values.put(outputSpecField,newOutputXML);
-        String newXML = jobDescription.getSpecification().toXML();
-        values.put(documentSpecField,newXML);
-        values.put(typeField,typeToString(jobDescription.getType()));
-        values.put(startMethodField,startMethodToString(jobDescription.getStartMethod()));
-        values.put(intervalField,jobDescription.getInterval());
-        values.put(reseedIntervalField,jobDescription.getReseedInterval());
-        values.put(expirationField,jobDescription.getExpiration());
-        values.put(priorityField,new Integer(jobDescription.getPriority()));
-        values.put(hopcountModeField,hopcountModeToString(jobDescription.getHopcountMode()));
-
-        if (set.getRowCount() > 0)
+        ICacheHandle ch = cacheManager.enterCache(null,invKeys,getTransactionID());
+        try
         {
-          // Update
-          // We need to reset the lastCheckTimeField if there are any changes that
-          // could affect what set of documents we allow!!!
-
-          IResultRow row = set.getRow(0);
-
-          boolean isSame = true;
-
-          // Determine whether we need to reset the scan time for documents.
-          // Basically, any change to job parameters that could affect ingestion should clear
isSame so that we
-          // relook at all the documents, not just the recent ones.
+          beginTransaction();
+          try
+          {
+            //performLock();
+            // See whether the instance exists
+            ArrayList params = new ArrayList();
+            String query = buildConjunctionClause(params,new ClauseDescription[]{
+              new UnitaryClause(idField,id)});
+            IResultSet set = performQuery("SELECT * FROM "+getTableName()+" WHERE "+
+              query+" FOR UPDATE",params,null,null);
+            HashMap values = new HashMap();
+            values.put(descriptionField,jobDescription.getDescription());
+            values.put(outputNameField,jobDescription.getOutputConnectionName());
+            values.put(connectionNameField,jobDescription.getConnectionName());
+            String newOutputXML = jobDescription.getOutputSpecification().toXML();
+            values.put(outputSpecField,newOutputXML);
+            String newXML = jobDescription.getSpecification().toXML();
+            values.put(documentSpecField,newXML);
+            values.put(typeField,typeToString(jobDescription.getType()));
+            values.put(startMethodField,startMethodToString(jobDescription.getStartMethod()));
+            values.put(intervalField,jobDescription.getInterval());
+            values.put(reseedIntervalField,jobDescription.getReseedInterval());
+            values.put(expirationField,jobDescription.getExpiration());
+            values.put(priorityField,new Integer(jobDescription.getPriority()));
+            values.put(hopcountModeField,hopcountModeToString(jobDescription.getHopcountMode()));
+
+            if (set.getRowCount() > 0)
+            {
+              // Update
+              // We need to reset the lastCheckTimeField if there are any changes that
+              // could affect what set of documents we allow!!!
+
+              IResultRow row = set.getRow(0);
+
+              boolean isSame = true;
+
+              // Determine whether we need to reset the scan time for documents.
+              // Basically, any change to job parameters that could affect ingestion should
clear isSame so that we
+              // relook at all the documents, not just the recent ones.
+
+              if (isSame)
+              {
+                String oldOutputSpecXML = (String)row.getValue(outputSpecField);
+                if (!oldOutputSpecXML.equals(newOutputXML))
+                  isSame = false;
+              }
+
+              if (isSame)
+              {
+                String oldDocSpecXML = (String)row.getValue(documentSpecField);
+                if (!oldDocSpecXML.equals(newXML))
+                  isSame = false;
+              }
+
+              if (!isSame)
+                values.put(lastCheckTimeField,null);
+
+              params.clear();
+              query = buildConjunctionClause(params,new ClauseDescription[]{
+                new UnitaryClause(idField,id)});
+              performUpdate(values," WHERE "+query,params,null);
+              scheduleManager.deleteRows(id);
+              hopFilterManager.deleteRows(id);
+            }
+            else
+            {
+              // Insert
+              values.put(startTimeField,null);
+              values.put(lastCheckTimeField,null);
+              values.put(endTimeField,null);
+              values.put(statusField,statusToString(STATUS_INACTIVE));
+              values.put(lastTimeField,new Long(System.currentTimeMillis()));
+              values.put(idField,id);
+              performInsert(values,null);
+            }
+
+            // Write schedule records
+            scheduleManager.writeRows(id,jobDescription);
+            // Write hop filter rows
+            hopFilterManager.writeRows(id,jobDescription);
 
-          if (isSame)
+            cacheManager.invalidateKeys(ch);
+            break;
+          }
+          catch (ManifoldCFException e)
           {
-            String oldOutputSpecXML = (String)row.getValue(outputSpecField);
-            if (!oldOutputSpecXML.equals(newOutputXML))
-              isSame = false;
+            signalRollback();
+            throw e;
           }
-
-          if (isSame)
+          catch (Error e)
           {
-            String oldDocSpecXML = (String)row.getValue(documentSpecField);
-            if (!oldDocSpecXML.equals(newXML))
-              isSame = false;
+            signalRollback();
+            throw e;
+          }
+          finally
+          {
+            endTransaction();
           }
-
-          if (!isSame)
-            values.put(lastCheckTimeField,null);
-
-          params.clear();
-          query = buildConjunctionClause(params,new ClauseDescription[]{
-            new UnitaryClause(idField,id)});
-          performUpdate(values," WHERE "+query,params,null);
-          scheduleManager.deleteRows(id);
-          hopFilterManager.deleteRows(id);
         }
-        else
+        finally
         {
-          // Insert
-          values.put(startTimeField,null);
-          values.put(lastCheckTimeField,null);
-          values.put(endTimeField,null);
-          values.put(statusField,statusToString(STATUS_INACTIVE));
-          values.put(lastTimeField,new Long(System.currentTimeMillis()));
-          values.put(idField,id);
-          performInsert(values,null);
+          cacheManager.leaveCache(ch);
         }
-
-        // Write schedule records
-        scheduleManager.writeRows(id,jobDescription);
-        // Write hop filter rows
-        hopFilterManager.writeRows(id,jobDescription);
-
-        cacheManager.invalidateKeys(ch);
       }
       catch (ManifoldCFException e)
       {
-        signalRollback();
-        throw e;
-      }
-      catch (Error e)
-      {
-        signalRollback();
-        throw e;
+        if (e.getErrorCode() != ManifoldCFException.DATABASE_TRANSACTION_ABORT)
+          throw e;
+        sleepAmt = getSleepAmt();
+        continue;
       }
       finally
       {
-        endTransaction();
+        sleepFor(sleepAmt);
       }
     }
-    finally
-    {
-      cacheManager.leaveCache(ch);
-    }
   }
 
   /** This method is called on a restart.

Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java?rev=1221745&r1=1221744&r2=1221745&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java
(original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/repository/RepositoryConnectionManager.java
Wed Dec 21 14:41:59 2011
@@ -355,7 +355,7 @@ public class RepositoryConnectionManager
           beginTransaction();
           try
           {
-            performLock();
+            //performLock();
             // Notify of a change to the configuration
             ManifoldCF.noteConfigurationChange();
             boolean isNew = object.getIsNew();



Mime
View raw message