manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1611211 - in /manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler: jobs/JobManager.java system/WorkerThread.java
Date Wed, 16 Jul 2014 22:26:30 GMT
Author: kwright
Date: Wed Jul 16 22:26:29 2014
New Revision: 1611211

URL: http://svn.apache.org/r1611211
Log:
Minor refactoring improvements

Modified:
    manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
    manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1611211&r1=1611210&r2=1611211&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Wed Jul 16 22:26:29 2014
@@ -3071,16 +3071,14 @@ public class JobManager implements IJobM
     // order the "select for update" operations appropriately.
     //
 
-    HashMap indexMap = new HashMap();
+    Map<String,Integer> indexMap = new HashMap<String,Integer>();
     String[] docIDHashes = new String[documentDescriptions.length];
 
-    int i = 0;
-    while (i < documentDescriptions.length)
+    for (int i = 0; i < documentDescriptions.length; i++)
     {
       String documentIDHash = documentDescriptions[i].getDocumentIdentifierHash() + ":" +
documentDescriptions[i].getJobID();
       docIDHashes[i] = documentIDHash;
       indexMap.put(documentIDHash,new Integer(i));
-      i++;
     }
 
     java.util.Arrays.sort(docIDHashes);
@@ -3095,13 +3093,10 @@ public class JobManager implements IJobM
       try
       {
         // Do one row at a time, to avoid deadlocking things
-        i = 0;
-        while (i < docIDHashes.length)
+        for (String docIDHash : docIDHashes)
         {
-          String docIDHash = docIDHashes[i];
-
           // Get the DocumentDescription object
-          DocumentDescription dd = documentDescriptions[((Integer)indexMap.get(docIDHash)).intValue()];
+          DocumentDescription dd = documentDescriptions[indexMap.get(docIDHash).intValue()];
 
           // Query for the status
           ArrayList list = new ArrayList();
@@ -3119,10 +3114,10 @@ public class JobManager implements IJobM
             // Update the jobqueue table
             jobQueue.updateCompletedRecord(dd.getID(),status);
           }
-          i++;
         }
         TrackerClass.notePrecommit();
         database.performCommit();
+        System.out.println("Committed updated status");
         TrackerClass.noteCommit();
         break;
       }
@@ -3146,6 +3141,12 @@ public class JobManager implements IJobM
         TrackerClass.noteRollback();
         throw e;
       }
+      catch (RuntimeException e)
+      {
+        database.signalRollback();
+        TrackerClass.noteRollback();
+        throw e;
+      }
       finally
       {
         database.endTransaction();
@@ -5037,7 +5038,7 @@ public class JobManager implements IJobM
           " docs and hopcounts for job "+jobID.toString()+" parent identifier hash "+parentIdentifierHash);
 
         // Go through document id's one at a time, in order - mainly to prevent deadlock
as much as possible.  Search for any existing row in jobqueue first (for update)
-        HashMap existingRows = new HashMap();
+        Map<String,JobqueueRecord> existingRows = new HashMap<String,JobqueueRecord>();
 
         for (int z = 0; z < reorderedDocIDHashes.length; z++)
         {
@@ -5091,7 +5092,7 @@ public class JobManager implements IJobM
         for (int z = 0; z < reorderedDocIDHashes.length; z++)
         {
           String docIDHash = reorderedDocIDHashes[z];
-          JobqueueRecord jr = (JobqueueRecord)existingRows.get(docIDHash);
+          JobqueueRecord jr = existingRows.get(docIDHash);
           if (jr != null)
           {
             // It was an existing row; do the update logic
@@ -5112,6 +5113,7 @@ public class JobManager implements IJobM
 
         TrackerClass.notePrecommit();
         database.performCommit();
+        System.out.println("Committed carrydown updates");
         TrackerClass.noteCommit();
         
         if (Logging.perf.isDebugEnabled())

Modified: manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java?rev=1611211&r1=1611210&r2=1611211&view=diff
==============================================================================
--- manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
(original)
+++ manifoldcf/trunk/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/WorkerThread.java
Wed Jul 16 22:26:29 2014
@@ -418,6 +418,7 @@ public class WorkerThread extends Thread
                         }
                         else if (serviceInterruption != null)
                         {
+
                           // Service interruption has precedence over unchanged, because
we might have been interrupted while scanning the document
                           // for references
                           DocumentDescription dd = qd.getDocumentDescription();
@@ -449,6 +450,7 @@ public class WorkerThread extends Thread
                         }
                         else if (activity.wasDocumentUnchanged(qd.getDocumentDescription().getDocumentIdentifier()))
                         {
+
                           finishList.add(qd);
                           ingesterCheckList.add(qd.getDocumentDescription().getDocumentIdentifierHash());
                         }
@@ -494,7 +496,7 @@ public class WorkerThread extends Thread
                         }
                         DocumentDescription[] requeueCandidates = jobManager.finishDocuments(job.getID(),legalLinkTypes,documentIDHashes,job.getHopcountMode());
                         ManifoldCF.requeueDocumentsDueToCarrydown(jobManager,requeueCandidates,connector,connection,rt,currentTime);
-
+                        
                         // In both job types, we have to go through the finishList to figure
out what to do with the documents.
                         // In the case of a document that was aborted, we must requeue it
for immediate reprocessing in BOTH job types.
                         switch (job.getType())
@@ -584,9 +586,8 @@ public class WorkerThread extends Thread
                             // Separate the ones we actually finished from the ones we need
to requeue because they were aborted
                             List<DocumentDescription> completedList = new ArrayList<DocumentDescription>();
                             List<DocumentDescription> abortedList = new ArrayList<DocumentDescription>();
-                            for (int i = 0; i < finishList.size(); i++)
+                            for (QueuedDocument qd : finishList)
                             {
-                              QueuedDocument qd = finishList.get(i);
                               DocumentDescription dd = qd.getDocumentDescription();
                               if (activity.wasDocumentAborted(dd.getDocumentIdentifier()))
                               {
@@ -622,7 +623,7 @@ public class WorkerThread extends Thread
                               DocumentDescription[] docDescriptions = new DocumentDescription[completedList.size()];
                               for (int i = 0; i < docDescriptions.length; i++)
                               {
-                                docDescriptions[i] = (DocumentDescription)completedList.get(i);
+                                docDescriptions[i] = completedList.get(i);
                               }
 
                               jobManager.markDocumentCompletedMultiple(docDescriptions);
@@ -672,9 +673,8 @@ public class WorkerThread extends Thread
               }
               
               // Handle rescanning
-              for (int i = 0; i < rescanList.size(); i++)
+              for (QueuedDocument qd : rescanList)
               {
-                QueuedDocument qd = rescanList.get(i);
                 jobManager.resetDocument(qd.getDocumentDescription(),0L,IJobManager.ACTION_RESCAN,-1L,-1);
                 qd.setProcessed();
               }
@@ -711,10 +711,9 @@ public class WorkerThread extends Thread
           {
             // Go through qds and requeue any that aren't closed out in one way or another.
 This allows the job
             // to be aborted; no dangling entries are left around.
-            int i = 0;
-            while (i < qds.getCount())
+            for (int i = 0; i < qds.getCount(); i++)
             {
-              QueuedDocument qd = qds.getDocument(i++);
+              QueuedDocument qd = qds.getDocument(i);
               if (!qd.wasProcessed())
               {
                 jobManager.resetDocument(qd.getDocumentDescription(),0L,IJobManager.ACTION_RESCAN,-1L,-1);



Mime
View raw message