manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1476809 - /manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
Date Sun, 28 Apr 2013 18:04:33 GMT
Author: kwright
Date: Sun Apr 28 18:04:32 2013
New Revision: 1476809

URL: http://svn.apache.org/r1476809
Log:
Clean up on beginFetch if the beginFetch operation is not complete.  Part of CONNECTORS-679.

Modified:
    manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java

Modified: manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java?rev=1476809&r1=1476808&r2=1476809&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
(original)
+++ manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
Sun Apr 28 18:04:32 2013
@@ -803,6 +803,16 @@ public class ThrottledFetcher
 
     }
 
+    /** Abort the fetch.
+    */
+    public void abortFetch()
+    {
+      synchronized (this)
+      {
+        refCount--;
+      }
+    }
+    
     /** Note the start of an individual byte read of a specified size.  Call this method
just before the
     * read request takes place.  Performs the necessary delay prior to reading specified
number of bytes from the server.
     */
@@ -1240,13 +1250,13 @@ public class ThrottledFetcher
     public void beginFetch(String fetchType)
       throws ManifoldCFException
     {
+      this.fetchType = fetchType;
+      this.fetchCounter = 0L;
+      int lastCreated = 0;
       try
       {
-        this.fetchType = fetchType;
-        this.fetchCounter = 0L;
         // Find or create the needed throttle bins
-        int i = 0;
-        while (i < throttleBinArray.length)
+        for (int i = 0; i < throttleBinArray.length; i++)
         {
           // Access the bins as we need them, and drop them when ref count goes to zero
           String binName = connectionBinArray[i].getBinName();
@@ -1262,13 +1272,23 @@ public class ThrottledFetcher
             tb.beginFetch();
           }
           throttleBinArray[i] = tb;
-          i++;
+          lastCreated = i + 1;
         }
       }
       catch (InterruptedException e)
       {
         throw new ManifoldCFException("Interrupted",ManifoldCFException.INTERRUPTED);
       }
+      finally
+      {
+        if (lastCreated != throttleBinArray.length)
+        {
+          for (int i = 0; i < lastCreated; i++)
+          {
+            throttleBinArray[i].abortFetch();
+          }
+        }
+      }
     }
 
     /** Execute the fetch and get the return code.  This method uses the



Mime
View raw message