Return-Path: X-Original-To: apmail-manifoldcf-commits-archive@www.apache.org Delivered-To: apmail-manifoldcf-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 20C5410AB5 for ; Fri, 5 Apr 2013 07:52:41 +0000 (UTC) Received: (qmail 93767 invoked by uid 500); 5 Apr 2013 07:52:40 -0000 Delivered-To: apmail-manifoldcf-commits-archive@manifoldcf.apache.org Received: (qmail 93717 invoked by uid 500); 5 Apr 2013 07:52:40 -0000 Mailing-List: contact commits-help@manifoldcf.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@manifoldcf.apache.org Delivered-To: mailing list commits@manifoldcf.apache.org Received: (qmail 93691 invoked by uid 99); 5 Apr 2013 07:52:39 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 05 Apr 2013 07:52:39 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 05 Apr 2013 07:52:35 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 7FD9D2388847; Fri, 5 Apr 2013 07:52:14 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1464874 - in /manifoldcf/branches/CONNECTORS-590-2: ./ framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/ Date: Fri, 05 Apr 2013 07:52:14 -0000 To: commits@manifoldcf.apache.org From: kwright@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130405075214.7FD9D2388847@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: kwright Date: Fri Apr 5 07:52:13 2013 New Revision: 1464874 URL: http://svn.apache.org/r1464874 Log: Port tracking code from CONNECTORS-590 to a more modern branch. Added: manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/TrackerClass.java - copied unchanged from r1464871, manifoldcf/branches/CONNECTORS-590/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/TrackerClass.java Modified: manifoldcf/branches/CONNECTORS-590-2/ (props changed) manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java Propchange: manifoldcf/branches/CONNECTORS-590-2/ ------------------------------------------------------------------------------ Merged /manifoldcf/branches/CONNECTORS-590:r1425803-1464871 Modified: manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1464874&r1=1464873&r2=1464874&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java (original) +++ manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java Fri Apr 5 07:52:13 2013 @@ -645,12 +645,14 @@ public class JobManager implements IJobM // Clean up carrydown stuff carryDown.reset(); database.performCommit(); + TrackerClass.noteCommit(); Logging.jobs.debug("Reset complete"); break; } catch (ManifoldCFException e) { database.signalRollback(); + TrackerClass.noteRollback(); if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT) { if (Logging.perf.isDebugEnabled()) @@ -663,6 +665,7 @@ public class JobManager implements IJobM catch (Error e) { database.signalRollback(); + TrackerClass.noteRollback(); throw e; } finally @@ -687,11 +690,13 @@ public class JobManager implements IJobM { jobQueue.resetDocumentWorkerStatus(); database.performCommit(); + TrackerClass.noteCommit(); break; } catch (ManifoldCFException e) { database.signalRollback(); + TrackerClass.noteRollback(); if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT) { if (Logging.perf.isDebugEnabled()) @@ -704,6 +709,7 @@ public class JobManager implements IJobM catch (Error e) { database.signalRollback(); + TrackerClass.noteRollback(); throw e; } finally @@ -732,6 +738,7 @@ public class JobManager implements IJobM { Logging.jobs.debug("Resetting doc deleting status"); jobQueue.resetDocDeleteWorkerStatus(); + TrackerClass.noteCommit(); Logging.jobs.debug("Reset complete"); } @@ -742,6 +749,7 @@ public class JobManager implements IJobM { Logging.jobs.debug("Resetting doc cleaning status"); jobQueue.resetDocCleanupWorkerStatus(); + TrackerClass.noteCommit(); Logging.jobs.debug("Reset complete"); } @@ -997,6 +1005,7 @@ public class JobManager implements IJobM } database.performCommit(); + TrackerClass.noteCommit(); if (Logging.perf.isDebugEnabled()) Logging.perf.debug("Done pruning unindexable docs after "+new Long(System.currentTimeMillis()-startTime).toString()+" ms."); @@ -1007,11 +1016,13 @@ public class JobManager implements IJobM catch (Error e) { database.signalRollback(); + TrackerClass.noteRollback(); throw e; } catch (ManifoldCFException e) { database.signalRollback(); + TrackerClass.noteRollback(); if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT) { if (Logging.perf.isDebugEnabled()) @@ -1258,6 +1269,7 @@ public class JobManager implements IJobM } database.performCommit(); + TrackerClass.noteCommit(); if (Logging.perf.isDebugEnabled()) Logging.perf.debug("Done pruning unindexable docs after "+new Long(System.currentTimeMillis()-startTime).toString()+" ms."); @@ -1268,11 +1280,13 @@ public class JobManager implements IJobM catch (Error e) { database.signalRollback(); + TrackerClass.noteRollback(); throw e; } catch (ManifoldCFException e) { database.signalRollback(); + TrackerClass.noteRollback(); if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT) { if (Logging.perf.isDebugEnabled()) @@ -1821,6 +1835,7 @@ public class JobManager implements IJobM } database.performCommit(); + TrackerClass.noteCommit(); return new DocumentSetAndFlags(rval, rvalBoolean); @@ -1828,6 +1843,7 @@ public class JobManager implements IJobM catch (ManifoldCFException e) { database.signalRollback(); + TrackerClass.noteRollback(); if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT) { if (Logging.perf.isDebugEnabled()) @@ -1840,6 +1856,7 @@ public class JobManager implements IJobM catch (Error e) { database.signalRollback(); + TrackerClass.noteRollback(); throw e; } finally @@ -2442,11 +2459,13 @@ public class JobManager implements IJobM i++; } database.performCommit(); + TrackerClass.noteCommit(); break; } catch (ManifoldCFException e) { database.signalRollback(); + TrackerClass.noteRollback(); if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT) { if (Logging.perf.isDebugEnabled()) @@ -2460,6 +2479,7 @@ public class JobManager implements IJobM catch (Error e) { database.signalRollback(); + TrackerClass.noteRollback(); throw e; } finally @@ -2607,11 +2627,13 @@ public class JobManager implements IJobM // we don't delete them here. database.performCommit(); + TrackerClass.noteCommit(); return rval; } catch (ManifoldCFException e) { database.signalRollback(); + TrackerClass.noteRollback(); if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT) { if (Logging.perf.isDebugEnabled()) @@ -2625,6 +2647,7 @@ public class JobManager implements IJobM catch (Error e) { database.signalRollback(); + TrackerClass.noteRollback(); throw e; } finally @@ -2987,16 +3010,19 @@ public class JobManager implements IJobM } database.performCommit(); + TrackerClass.noteCommit(); break; } catch (Error e) { database.signalRollback(); + TrackerClass.noteRollback(); throw e; } catch (ManifoldCFException e) { database.signalRollback(); + TrackerClass.noteRollback(); if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT) { if (Logging.perf.isDebugEnabled()) @@ -3194,11 +3220,13 @@ public class JobManager implements IJobM } database.performCommit(); + TrackerClass.noteCommit(); break; } catch (ManifoldCFException e) { database.signalRollback(); + TrackerClass.noteRollback(); if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT) { if (Logging.perf.isDebugEnabled()) @@ -3211,6 +3239,7 @@ public class JobManager implements IJobM catch (Error e) { database.signalRollback(); + TrackerClass.noteRollback(); throw e; } finally @@ -3289,11 +3318,13 @@ public class JobManager implements IJobM } database.performCommit(); + TrackerClass.noteCommit(); break; } catch (ManifoldCFException e) { database.signalRollback(); + TrackerClass.noteRollback(); if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT) { if (Logging.perf.isDebugEnabled()) @@ -3306,6 +3337,7 @@ public class JobManager implements IJobM catch (Error e) { database.signalRollback(); + TrackerClass.noteRollback(); throw e; } finally @@ -3524,6 +3556,7 @@ public class JobManager implements IJobM hopCount.recordSeedReferences(jobID,legalLinkTypes,reorderedDocIDHashes,hopcountMethod); database.performCommit(); + TrackerClass.noteCommit(); if (Logging.perf.isDebugEnabled()) Logging.perf.debug("Took "+new Long(System.currentTimeMillis()-startTime).toString()+" ms to add "+Integer.toString(reorderedDocIDHashes.length)+ @@ -3544,6 +3577,7 @@ public class JobManager implements IJobM catch (ManifoldCFException e) { database.signalRollback(); + TrackerClass.noteRollback(); if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT) { if (Logging.perf.isDebugEnabled()) @@ -3557,6 +3591,7 @@ public class JobManager implements IJobM catch (Error e) { database.signalRollback(); + TrackerClass.noteRollback(); throw e; } finally @@ -4154,6 +4189,7 @@ public class JobManager implements IJobM jobQueue.reactivateHopcountRemovedRecords(jobID); database.performCommit(); + TrackerClass.noteCommit(); if (Logging.perf.isDebugEnabled()) Logging.perf.debug("Took "+new Long(System.currentTimeMillis()-startTime).toString()+" ms to add "+Integer.toString(reorderedDocIDHashes.length)+ @@ -4173,6 +4209,7 @@ public class JobManager implements IJobM catch (ManifoldCFException e) { database.signalRollback(); + TrackerClass.noteRollback(); if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT) { sleepAmt = getRandomAmount(); @@ -4186,6 +4223,7 @@ public class JobManager implements IJobM catch (Error e) { database.signalRollback(); + TrackerClass.noteRollback(); throw e; } finally @@ -5377,6 +5415,7 @@ public class JobManager implements IJobM // No special treatment needed for hopcount or carrydown, since these all get deleted at once // at the end of the job delete process. jobQueue.prepareDeleteScan(jobID); + TrackerClass.noteCommit(); } /** Prepare a job to be run. @@ -5469,11 +5508,13 @@ public class JobManager implements IJobM jobQueue.prepareFullScan(jobID); database.performCommit(); + TrackerClass.noteCommit(); break; } catch (ManifoldCFException e) { database.signalRollback(); + TrackerClass.noteRollback(); if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT) { if (Logging.perf.isDebugEnabled()) @@ -5486,6 +5527,7 @@ public class JobManager implements IJobM catch (Error e) { database.signalRollback(); + TrackerClass.noteRollback(); throw e; } finally Modified: manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java?rev=1464874&r1=1464873&r2=1464874&view=diff ============================================================================== --- manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java (original) +++ manifoldcf/branches/CONNECTORS-590-2/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobQueue.java Fri Apr 5 07:52:13 2013 @@ -390,6 +390,8 @@ public class JobQueue extends org.apache // Reindex the jobqueue table, since we've probably made lots of bad tuples doing the above operations. reindexTable(); unconditionallyAnalyzeTables(); + + TrackerClass.noteGlobalEvent("Restart"); } /** Flip all records for a job that have status HOPCOUNTREMOVED back to PENDING. @@ -407,6 +409,8 @@ public class JobQueue extends org.apache new UnitaryClause(jobIDField,jobID), new UnitaryClause(statusField,statusToString(STATUS_HOPCOUNTREMOVED))}); performUpdate(map,"WHERE "+query,list,null); + + TrackerClass.noteJobEvent(jobID,"Map HOPCOUNTREMOVED to PENDING"); } /** Delete all records for a job that have status HOPCOUNTREMOVED. @@ -464,6 +468,8 @@ public class JobQueue extends org.apache statusToString(STATUS_ACTIVEPURGATORY), statusToString(STATUS_ACTIVENEEDRESCANPURGATORY)})}); performUpdate(map,"WHERE "+query,list,null); + + TrackerClass.noteGlobalEvent("Reset document worker status"); } /** Reset doc delete worker status. @@ -479,6 +485,8 @@ public class JobQueue extends org.apache String query = buildConjunctionClause(list,new ClauseDescription[]{ new UnitaryClause(statusField,statusToString(STATUS_BEINGDELETED))}); performUpdate(map,"WHERE "+query,list,null); + + TrackerClass.noteGlobalEvent("Reset document delete worker status"); } /** Reset doc cleaning worker status. @@ -494,6 +502,8 @@ public class JobQueue extends org.apache String query = buildConjunctionClause(list,new ClauseDescription[]{ new UnitaryClause(statusField,statusToString(STATUS_BEINGCLEANED))}); performUpdate(map,"WHERE "+query,list,null); + + TrackerClass.noteGlobalEvent("Reset document cleanup worker status"); } /** Prepare for a job delete pass. This will not be called @@ -545,6 +555,7 @@ public class JobQueue extends org.apache // Do an analyze, otherwise our plans are going to be crap right off the bat unconditionallyAnalyzeTables(); + TrackerClass.noteJobEvent(jobID,"Prepare delete scan"); } /** Prepare for a "full scan" job. This will not be called @@ -595,6 +606,8 @@ public class JobQueue extends org.apache noteModifications(0,2,0); // Do an analyze, otherwise our plans are going to be crap right off the bat unconditionallyAnalyzeTables(); + + TrackerClass.noteJobEvent(jobID,"Prepare full scan"); } /** Prepare for a "partial" job. This is called ONLY when the job is inactive. @@ -654,6 +667,8 @@ public class JobQueue extends org.apache noteModifications(0,1,0); // Do an analyze, otherwise our plans are going to be crap right off the bat unconditionallyAnalyzeTables(); + + TrackerClass.noteJobEvent(jobID,"Prepare incremental scan"); } /** Delete ingested document identifiers (as part of deleting the owning job). @@ -767,6 +782,7 @@ public class JobQueue extends org.apache // Leave doc priority unchanged. break; default: + TrackerClass.printForensics(recID, currentStatus); throw new ManifoldCFException("Unexpected jobqueue status - record id "+recID.toString()+", expecting active status, saw "+Integer.toString(currentStatus)); } @@ -779,6 +795,8 @@ public class JobQueue extends org.apache String query = buildConjunctionClause(list,new ClauseDescription[]{ new UnitaryClause(idField,recID)}); performUpdate(map,"WHERE "+query,list,null); + + TrackerClass.noteRecordEvent(recID, newStatus, "Note completion"); } /** Either mark a record as hopcountremoved, or set status to "rescan", depending on the @@ -814,6 +832,7 @@ public class JobQueue extends org.apache // Leave doc priority unchanged. break; default: + TrackerClass.printForensics(recID, currentStatus); throw new ManifoldCFException("Unexpected jobqueue status - record id "+recID.toString()+", expecting active status, saw "+Integer.toString(currentStatus)); } @@ -826,6 +845,7 @@ public class JobQueue extends org.apache String query = buildConjunctionClause(list,new ClauseDescription[]{ new UnitaryClause(idField,recID)}); performUpdate(map,"WHERE "+query,list,null); + TrackerClass.noteRecordEvent(recID, newStatus, "Update or hopcount remove"); return rval; } @@ -846,6 +866,7 @@ public class JobQueue extends org.apache newStatus = STATUS_ACTIVEPURGATORY; break; default: + TrackerClass.printForensics(id, currentStatus); throw new ManifoldCFException("Unexpected status value for jobqueue record "+id.toString()+"; got "+Integer.toString(currentStatus)); } @@ -856,6 +877,7 @@ public class JobQueue extends org.apache new UnitaryClause(idField,id)}); performUpdate(map,"WHERE "+query,list,null); noteModifications(0,1,0); + TrackerClass.noteRecordEvent(id, newStatus, "Make active"); } /** Set the status on a record, including check time and priority. @@ -887,6 +909,7 @@ public class JobQueue extends org.apache new UnitaryClause(idField,id)}); performUpdate(map,"WHERE "+query,list,null); noteModifications(0,1,0); + TrackerClass.noteRecordEvent(id, status, "Set status"); } /** Set the status of a document to "being deleted". @@ -901,6 +924,7 @@ public class JobQueue extends org.apache new UnitaryClause(idField,id)}); performUpdate(map,"WHERE "+query,list,null); noteModifications(0,1,0); + TrackerClass.noteRecordEvent(id, STATUS_BEINGDELETED, "Set deleting status"); } /** Set the status of a document to be "no longer deleting" */ @@ -918,6 +942,7 @@ public class JobQueue extends org.apache new UnitaryClause(idField,id)}); performUpdate(map,"WHERE "+query,list,null); noteModifications(0,1,0); + TrackerClass.noteRecordEvent(id, STATUS_ELIGIBLEFORDELETE, "Set undeleting status"); } /** Set the status of a document to "being cleaned". @@ -932,6 +957,7 @@ public class JobQueue extends org.apache new UnitaryClause(idField,id)}); performUpdate(map,"WHERE "+query,list,null); noteModifications(0,1,0); + TrackerClass.noteRecordEvent(id, STATUS_BEINGCLEANED, "Set cleaning status"); } /** Set the status of a document to be "no longer cleaning" */ @@ -949,6 +975,7 @@ public class JobQueue extends org.apache new UnitaryClause(idField,id)}); performUpdate(map,"WHERE "+query,list,null); noteModifications(0,1,0); + TrackerClass.noteRecordEvent(id, STATUS_PURGATORY, "Set uncleaning status"); } /** Remove multiple records entirely. @@ -1040,6 +1067,7 @@ public class JobQueue extends org.apache case STATUS_PURGATORY: // Set the status and time both map.put(statusField,statusToString(STATUS_PENDINGPURGATORY)); + TrackerClass.noteRecordEvent(recordID, STATUS_PENDINGPURGATORY, "Update existing record initial"); if (desiredExecuteTime == -1L) map.put(checkTimeField,new Long(0L)); else @@ -1130,6 +1158,7 @@ public class JobQueue extends org.apache performInsert(map,null); prereqEventManager.addRows(recordID,prereqEvents); noteModifications(1,0,0); + TrackerClass.noteRecordEvent(recordID, STATUS_PENDING, "Create initial"); } /** Note the remaining documents that do NOT need to be queued. These are noted so that the @@ -1317,6 +1346,7 @@ public class JobQueue extends org.apache case STATUS_UNCHANGED: // Set the status and time both map.put(statusField,statusToString(STATUS_PENDINGPURGATORY)); + TrackerClass.noteRecordEvent(recordID, STATUS_PENDINGPURGATORY, "Update existing"); map.put(checkTimeField,new Long(desiredExecuteTime)); map.put(checkActionField,actionToString(ACTION_RESCAN)); map.put(failTimeField,null); @@ -1334,6 +1364,7 @@ public class JobQueue extends org.apache // The document has been processed before, so it has to go into PENDINGPURGATORY. // Set the status and time both map.put(statusField,statusToString(STATUS_PENDINGPURGATORY)); + TrackerClass.noteRecordEvent(recordID, STATUS_PENDINGPURGATORY, "Update existing"); map.put(checkTimeField,new Long(desiredExecuteTime)); map.put(checkActionField,actionToString(ACTION_RESCAN)); map.put(failTimeField,null); @@ -1363,6 +1394,7 @@ public class JobQueue extends org.apache // Flip the state to the new one, and set the document priority at this time too - it will be preserved when the // processing is completed. map.put(statusField,statusToString(STATUS_ACTIVENEEDRESCAN)); + TrackerClass.noteRecordEvent(recordID, STATUS_ACTIVENEEDRESCAN, "Update existing"); map.put(checkTimeField,new Long(desiredExecuteTime)); map.put(checkActionField,actionToString(ACTION_RESCAN)); map.put(failTimeField,null); @@ -1387,6 +1419,7 @@ public class JobQueue extends org.apache // Flip the state to the new one, and set the document priority at this time too - it will be preserved when the // processing is completed. map.put(statusField,statusToString(STATUS_ACTIVENEEDRESCANPURGATORY)); + TrackerClass.noteRecordEvent(recordID, STATUS_ACTIVENEEDRESCANPURGATORY, "Update existing"); map.put(checkTimeField,new Long(desiredExecuteTime)); map.put(checkActionField,actionToString(ACTION_RESCAN)); map.put(failTimeField,null); @@ -1454,6 +1487,8 @@ public class JobQueue extends org.apache performInsert(map,null); prereqEventManager.addRows(recordID,prereqEvents); noteModifications(1,0,0); + TrackerClass.noteRecordEvent(recordID, STATUS_PENDING, "Create new"); + } // Methods to convert status strings to integers and back