manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1209309 - in /incubator/lcf/trunk/framework: agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/ core/src/main/java/org/apache/manifoldcf/core/database/ pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/
Date Thu, 01 Dec 2011 23:37:33 GMT
Author: kwright
Date: Thu Dec  1 23:37:32 2011
New Revision: 1209309

URL: http://svn.apache.org/viewvc?rev=1209309&view=rev
Log:
Fix for CONNECTORS-299.  Do the commits early in all transactions where deadlock is a possibility.

Modified:
    incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
    incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
    incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java

Modified: incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java?rev=1209309&r1=1209308&r2=1209309&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
(original)
+++ incubator/lcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
Thu Dec  1 23:37:32 2011
@@ -1393,10 +1393,12 @@ public class IncrementalIngester extends
             map.put(changeCountField,new Long(changeCount));
             performUpdate(map,"WHERE "+query,list,null);
             // Update successful!
+            performCommit();
             return;
           }
 
           // Update failed to find a matching record, so try the insert
+          performCommit();
           break;
         }
         catch (ManifoldCFException e)

Modified: incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java?rev=1209309&r1=1209308&r2=1209309&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
(original)
+++ incubator/lcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/Database.java
Thu Dec  1 23:37:32 2011
@@ -370,8 +370,7 @@ public abstract class Database
       }
       else
       {
-        if (!commitDone)
-          cacheManager.commitTransaction(th.getTransactionID());
+	cacheManager.commitTransaction(th.getTransactionID());
       }
       commitDone = false;
       doRollback = false;

Modified: incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1209309&r1=1209308&r2=1209309&view=diff
==============================================================================
--- incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
(original)
+++ incubator/lcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Thu Dec  1 23:37:32 2011
@@ -638,6 +638,7 @@ public class JobManager implements IJobM
         hopCount.reset();
         // Clean up carrydown stuff
         carryDown.reset();
+        database.performCommit();
         Logging.jobs.debug("Reset complete");
         break;
       }
@@ -679,6 +680,7 @@ public class JobManager implements IJobM
       try
       {
         jobQueue.resetDocumentWorkerStatus();
+        database.performCommit();
         break;
       }
       catch (ManifoldCFException e)
@@ -987,7 +989,9 @@ public class JobManager implements IJobM
           rval[i++] = dd;
           jobQueue.setCleaningStatus(dd.getID());
         }
-
+        
+        database.performCommit();
+        
         if (Logging.perf.isDebugEnabled())
           Logging.perf.debug("Done pruning unindexable docs after "+new Long(System.currentTimeMillis()-startTime).toString()+"
ms.");
 
@@ -1247,6 +1251,8 @@ public class JobManager implements IJobM
           i++;
         }
 
+        database.performCommit();
+        
         if (Logging.perf.isDebugEnabled())
           Logging.perf.debug("Done pruning unindexable docs after "+new Long(System.currentTimeMillis()-startTime).toString()+"
ms.");
 
@@ -1569,6 +1575,7 @@ public class JobManager implements IJobM
             Logging.perf.debug("Setting document priority for '"+dd.getDocumentIdentifier()+"'
to "+new Double(priority).toString()+", set time "+new Long(currentTime).toString());
           i++;
         }
+        database.performCommit();
         break;
       }
       catch (ManifoldCFException e)
@@ -1803,6 +1810,8 @@ public class JobManager implements IJobM
           jobQueue.updateActiveRecord(dd.getID(),((Integer)statusMap.get(compositeDocID)).intValue());
         }
 
+        database.performCommit();
+        
         return new DocumentSetAndFlags(rval, rvalBoolean);
 
       }
@@ -2310,6 +2319,7 @@ public class JobManager implements IJobM
 
             i++;
           }
+          database.performCommit();
           break;
         }
         catch (ManifoldCFException e)
@@ -2425,6 +2435,7 @@ public class JobManager implements IJobM
           }
           i++;
         }
+        database.performCommit();
         break;
       }
       catch (ManifoldCFException e)
@@ -2536,6 +2547,8 @@ public class JobManager implements IJobM
         if (legalLinkTypes.length > 0)
           hopCount.deleteDocumentIdentifiers(jobID,legalLinkTypes,docIDSimpleHashes,hopcountMethod);
 
+        database.performCommit();
+        
         if (Logging.perf.isDebugEnabled())
           Logging.perf.debug("Took "+new Long(System.currentTimeMillis()-startTime).toString()+"
ms to delete "+Integer.toString(docIDHashes.length)+
           " docs and clean up hopcount for job "+jobID.toString());
@@ -2744,6 +2757,7 @@ public class JobManager implements IJobM
           i++;
         }
 
+        database.performCommit();
         break;
       }
       catch (Error e)
@@ -2866,6 +2880,7 @@ public class JobManager implements IJobM
           i++;
         }
 
+        database.performCommit();
         break;
       }
       catch (Error e)
@@ -2949,6 +2964,7 @@ public class JobManager implements IJobM
           i++;
         }
 
+        database.performCommit();
         break;
       }
       catch (ManifoldCFException e)
@@ -3043,6 +3059,7 @@ public class JobManager implements IJobM
           i++;
         }
 
+        database.performCommit();
         break;
       }
       catch (ManifoldCFException e)
@@ -3277,6 +3294,8 @@ public class JobManager implements IJobM
         if (legalLinkTypes.length > 0)
           hopCount.recordSeedReferences(jobID,legalLinkTypes,reorderedDocIDHashes,hopcountMethod);
 
+        database.performCommit();
+        
         if (Logging.perf.isDebugEnabled())
           Logging.perf.debug("Took "+new Long(System.currentTimeMillis()-startTime).toString()+"
ms to add "+Integer.toString(reorderedDocIDHashes.length)+
           " initial docs and hopcounts for job "+jobID.toString());
@@ -3360,6 +3379,8 @@ public class JobManager implements IJobM
         if (legalLinkTypes.length > 0)
           hopCount.recordSeedReferences(jobID,legalLinkTypes,reorderedDocIDHashes,hopcountMethod);
 
+        database.performCommit();
+        
         if (Logging.perf.isDebugEnabled())
           Logging.perf.debug("Took "+new Long(System.currentTimeMillis()-startTime).toString()+"
ms to add "+Integer.toString(reorderedDocIDHashes.length)+
           " remaining docs and hopcounts for job "+jobID.toString());
@@ -3432,6 +3453,8 @@ public class JobManager implements IJobM
         if (legalLinkTypes.length > 0)
           hopCount.finishSeedReferences(jobID,legalLinkTypes,hopcountMethod);
 
+        database.performCommit();
+        
         if (Logging.perf.isDebugEnabled())
           Logging.perf.debug("Took "+new Long(System.currentTimeMillis()-startTime).toString()+
           " ms to finish initial docs and hopcounts for job "+jobID.toString());
@@ -3590,11 +3613,13 @@ public class JobManager implements IJobM
           {
             // Sleep a little bit so another thread can have a whack at things
             sleepAmt = 100L;
+            database.performCommit();
             continue;
           }
 
           // Definitive answers found; continue through.
           distances = hopCount.findHopCounts(jobID,askDocIDHashes,linkType);
+          database.performCommit();
         }
         catch (ManifoldCFException e)
         {
@@ -3887,6 +3912,8 @@ public class JobManager implements IJobM
         if (parentIdentifierHash != null && relationshipType != null)
           hopCount.recordReferences(jobID,legalLinkTypes,parentIdentifierHash,reorderedDocIDHashes,relationshipType,hopcountMethod);
 
+        database.performCommit();
+        
         if (Logging.perf.isDebugEnabled())
           Logging.perf.debug("Took "+new Long(System.currentTimeMillis()-startTime).toString()+"
ms to add "+Integer.toString(reorderedDocIDHashes.length)+
           " docs and hopcounts for job "+jobID.toString()+" parent identifier hash "+parentIdentifierHash);
@@ -4303,6 +4330,7 @@ public class JobManager implements IJobM
             0L,currentTime,true,docPriorities[originalIndex],null);
           j++;
         }
+        database.performCommit();
         break;
       }
       catch (ManifoldCFException e)
@@ -5108,6 +5136,7 @@ public class JobManager implements IJobM
         }
 
         jobQueue.prepareFullScan(jobID);
+        database.performCommit();
         break;
       }
       catch (ManifoldCFException e)
@@ -5323,6 +5352,7 @@ public class JobManager implements IJobM
           rval[i] = new JobStartRecord(jobID,synchTime);
           i++;
         }
+        database.performCommit();
         return rval;
       }
       catch (ManifoldCFException e)
@@ -5390,6 +5420,7 @@ public class JobManager implements IJobM
           rval[i] = new JobStartRecord(jobID,0L);
           i++;
         }
+        database.performCommit();
         return rval;
       }
       catch (ManifoldCFException e)
@@ -5463,6 +5494,7 @@ public class JobManager implements IJobM
           rval[i] = new JobStartRecord(jobID,synchTime);
           i++;
         }
+        database.performCommit();
         return rval;
       }
       catch (ManifoldCFException e)
@@ -5527,6 +5559,7 @@ public class JobManager implements IJobM
         default:
           throw new ManifoldCFException("Unexpected job status: "+Integer.toString(status));
         }
+        database.performCommit();
         return;
       }
       catch (ManifoldCFException e)
@@ -5594,6 +5627,7 @@ public class JobManager implements IJobM
         default:
           throw new ManifoldCFException("Unexpected job status: "+Integer.toString(status));
         }
+        database.performCommit();
         return;
       }
       catch (ManifoldCFException e)
@@ -5661,6 +5695,7 @@ public class JobManager implements IJobM
         default:
           throw new ManifoldCFException("Unexpected job status: "+Integer.toString(status));
         }
+        database.performCommit();
         return;
       }
       catch (ManifoldCFException e)
@@ -5743,6 +5778,7 @@ public class JobManager implements IJobM
         default:
           throw new ManifoldCFException("Unexpected job status: "+Integer.toString(status));
         }
+        database.performCommit();
         return;
       }
       catch (ManifoldCFException e)
@@ -5877,6 +5913,7 @@ public class JobManager implements IJobM
         default:
           throw new ManifoldCFException("Unexpected job status: "+Integer.toString(status));
         }
+        database.performCommit();
         return;
       }
       catch (ManifoldCFException e)
@@ -5980,6 +6017,7 @@ public class JobManager implements IJobM
             Logging.jobs.debug("Removed job "+jobID);
           }
         }
+        database.performCommit();
         return;
       }
       catch (ManifoldCFException e)
@@ -6083,6 +6121,7 @@ public class JobManager implements IJobM
           }
 
         }
+        database.performCommit();
         return;
       }
       catch (ManifoldCFException e)
@@ -6147,6 +6186,7 @@ public class JobManager implements IJobM
           }
           rval[i++] = new JobStartRecord(jobID,0L);
         }
+        database.performCommit();
         return rval;
       }
       catch (ManifoldCFException e)



Mime
View raw message