manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1476354 - in /manifoldcf/trunk/connectors: rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/
Date Fri, 26 Apr 2013 19:36:04 GMT
Author: kwright
Date: Fri Apr 26 19:36:03 2013
New Revision: 1476354

URL: http://svn.apache.org/r1476354
Log:
Another fix for throttling (CONNECTORS-679).  Use one synchronizer instead of two.  This may
make some things slightly slower, but I believe our current use allowed unsynchronized variables
to be modified.

Modified:
    manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
    manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java

Modified: manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java?rev=1476354&r1=1476353&r2=1476354&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
(original)
+++ manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
Fri Apr 26 19:36:03 2013
@@ -978,9 +978,6 @@ public class ThrottledFetcher
     /** Total actual bytes read in this series; this includes fetches in progress */
     protected long totalBytesRead = -1L;
 
-    /** This object is used to gate access while the first chunk is being read */
-    protected Integer firstChunkLock = new Integer(0);
-
     /** Outstanding connection counter */
     protected volatile int outstandingConnections = 0;
 
@@ -1102,19 +1099,16 @@ public class ThrottledFetcher
 
       long currentTime = System.currentTimeMillis();
 
-      synchronized (firstChunkLock)
+      synchronized (this)
       {
         while (estimateInProgress)
-          firstChunkLock.wait();
+          wait();
         if (estimateValid == false)
         {
           seriesStartTime = currentTime;
           estimateInProgress = true;
           // Add these bytes to the estimated total
-          synchronized (this)
-          {
-            totalBytesRead += (long)byteCount;
-          }
+          totalBytesRead += (long)byteCount;
           // Exit early; this thread isn't going to do any waiting
           //if (Logging.connectors.isTraceEnabled())
           //      Logging.connectors.trace("RSS: Read begin noted; gathering stats for '"+serverName+"'");
@@ -1161,11 +1155,21 @@ public class ThrottledFetcher
       {
         if (!finished)
         {
-          if (estimateInProgress)
-          {
-            estimateInProgress = false;
-            firstChunkLock.notifyAll();
-          }
+          abortRead();
+        }
+      }
+    }
+
+    /** Abort a read in progress.
+    */
+    public void abortRead()
+    {
+      synchronized (this)
+      {
+        if (estimateInProgress)
+        {
+          estimateInProgress = false;
+          notifyAll();
         }
       }
     }
@@ -1183,11 +1187,6 @@ public class ThrottledFetcher
       synchronized (this)
       {
         totalBytesRead = totalBytesRead + (long)actualCount - (long)originalCount;
-      }
-
-      // Only one thread should get here if it's the first chunk, but we synchronize to be
sure
-      synchronized (firstChunkLock)
-      {
         if (estimateInProgress)
         {
           if (actualCount == 0)
@@ -1197,7 +1196,7 @@ public class ThrottledFetcher
             rateEstimate = ((double)(currentTime - seriesStartTime))/(double)actualCount;
           estimateValid = true;
           estimateInProgress = false;
-          firstChunkLock.notifyAll();
+          notifyAll();
         }
       }
 

Modified: manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java?rev=1476354&r1=1476353&r2=1476354&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
(original)
+++ manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
Fri Apr 26 19:36:03 2013
@@ -769,9 +769,6 @@ public class ThrottledFetcher
     /** Total actual bytes read in this series; this includes fetches in progress */
     protected long totalBytesRead = -1L;
 
-    /** This object is used to gate access while the first chunk is being read */
-    protected Integer firstChunkLock = new Integer(0);
-
     /** Constructor. */
     public ThrottleBin(String binName)
     {
@@ -814,19 +811,16 @@ public class ThrottledFetcher
     {
       long currentTime = System.currentTimeMillis();
 
-      synchronized (firstChunkLock)
+      synchronized (this)
       {
         while (estimateInProgress)
-          firstChunkLock.wait();
+          wait();
         if (estimateValid == false)
         {
           seriesStartTime = currentTime;
           estimateInProgress = true;
           // Add these bytes to the estimated total
-          synchronized (this)
-          {
-            totalBytesRead += (long)byteCount;
-          }
+          totalBytesRead += (long)byteCount;
           // Exit early; this thread isn't going to do any waiting
           return;
         }
@@ -867,15 +861,25 @@ public class ThrottledFetcher
       {
         if (!finished)
         {
-          if (estimateInProgress)
-          {
-            estimateInProgress = false;
-            firstChunkLock.notifyAll();
-          }
+          abortRead();
         }
       }
     }
 
+    /** Abort a read in progress.
+    */
+    public void abortRead()
+    {
+      synchronized (this)
+      {
+        if (estimateInProgress)
+        {
+          estimateInProgress = false;
+          notifyAll();
+        }
+      }
+    }
+    
     /** Note the end of an individual read from the server.  Call this just after an individual
read completes.
     * Pass the actual number of bytes read to the method.
     */
@@ -886,11 +890,6 @@ public class ThrottledFetcher
       synchronized (this)
       {
         totalBytesRead = totalBytesRead + (long)actualCount - (long)originalCount;
-      }
-
-      // Only one thread should get here if it's the first chunk, but we synchronize to be
sure
-      synchronized (firstChunkLock)
-      {
         if (estimateInProgress)
         {
           if (actualCount == 0)
@@ -900,7 +899,7 @@ public class ThrottledFetcher
             rateEstimate = ((double)(currentTime - seriesStartTime))/(double)actualCount;
           estimateValid = true;
           estimateInProgress = false;
-          firstChunkLock.notifyAll();
+          notifyAll();
         }
       }
     }
@@ -1179,11 +1178,24 @@ public class ThrottledFetcher
       throws InterruptedException
     {
       // Consult with throttle bins
-      int i = 0;
-      while (i < throttleBinArray.length)
+      int lastOneDone = 0;
+      try
       {
-        throttleBinArray[i].beginRead(len,minMillisecondsPerByte[i]);
-        i++;
+        for (int i = 0; i < throttleBinArray.length; i++)
+        {
+          throttleBinArray[i].beginRead(len,minMillisecondsPerByte[i]);
+          lastOneDone = i + 1;
+        }
+      }
+      finally
+      {
+        if (lastOneDone != throttleBinArray.length)
+        {
+          for (int i = 0; i < lastOneDone; i++)
+          {
+            throttleBinArray[i].abortRead();
+          }
+        }
       }
     }
 
@@ -1191,11 +1203,9 @@ public class ThrottledFetcher
     public void endRead(int origLen, int actualAmt)
     {
       // Consult with throttle bins
-      int i = 0;
-      while (i < throttleBinArray.length)
+      for (int i = 0; i < throttleBinArray.length; i++)
       {
         throttleBinArray[i].endRead(origLen,actualAmt);
-        i++;
       }
     }
 



Mime
View raw message