manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1489745 - in /manifoldcf/trunk/connectors: dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/ googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/
Date Wed, 05 Jun 2013 08:54:00 GMT
Author: kwright
Date: Wed Jun  5 08:53:59 2013
New Revision: 1489745

URL: http://svn.apache.org/r1489745
Log:
One more improvement: If interrupted, don't wait for background thread.

Modified:
    manifoldcf/trunk/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java
    manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java

Modified: manifoldcf/trunk/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java?rev=1489745&r1=1489744&r2=1489745&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java
(original)
+++ manifoldcf/trunk/connectors/dropbox/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/dropbox/DropboxRepositoryConnector.java
Wed Jun  5 08:53:59 2013
@@ -676,6 +676,7 @@ public class DropboxRepositoryConnector 
     GetSeedsThread t = new GetSeedsThread(dropboxPath);
     try {
       t.start();
+      boolean wasInterrupted = false;
       try {
         XThreadStringBuffer seedBuffer = t.getBuffer();
 
@@ -688,8 +689,16 @@ public class DropboxRepositoryConnector 
           // Add the pageID to the queue
           activities.addSeedDocument(docPath);
         }
+      } catch (InterruptedException e) {
+        wasInterrupted = true;
+        throw e;
+      } catch (ManifoldCFException e) {
+        if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+          wasInterrupted = true;
+        throw e;
       } finally {
-        t.finishUp();
+        if (!wasInterrupted)
+          t.finishUp();
       }
     } catch (InterruptedException e) {
       t.interrupt();
@@ -852,6 +861,7 @@ public class DropboxRepositoryConnector 
             BackgroundStreamThread t = new BackgroundStreamThread(nodeId);
             try {
               t.start();
+              boolean wasInterrupted = false;
               try {
                 InputStream is = t.getSafeInputStream();
                 try {
@@ -860,9 +870,19 @@ public class DropboxRepositoryConnector 
                 } finally {
                   is.close();
                 }
+              } catch (java.net.SocketTimeoutException e) {
+                throw e;
+              } catch (InterruptedIOException e) {
+                wasInterrupted = true;
+                throw e;
+              } catch (ManifoldCFException e) {
+                if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+                  wasInterrupted = true;
+                throw e;
               } finally {
-                // This does a join
-                t.finishUp();
+                if (!wasInterrupted)
+                  // This does a join
+                  t.finishUp();
               }
 
               // No errors.  Record the fact that we made it.
@@ -874,6 +894,10 @@ public class DropboxRepositoryConnector 
               t.interrupt();
               throw new ManifoldCFException("Interrupted: " + e.getMessage(), e,
                 ManifoldCFException.INTERRUPTED);
+            } catch (java.net.SocketTimeoutException e) {
+              errorCode = "IO ERROR";
+              errorDesc = e.getMessage();
+              handleIOException(e);
             } catch (InterruptedIOException e) {
               t.interrupt();
               throw new ManifoldCFException("Interrupted: " + e.getMessage(), e,
@@ -1111,7 +1135,7 @@ public class DropboxRepositoryConnector 
   /** Handle an IO exception. */
   protected static void handleIOException(IOException e)
     throws ManifoldCFException, ServiceInterruption {
-    if (e instanceof InterruptedIOException) {
+    if (!(e instanceof java.net.SocketTimeoutException) && (e instanceof InterruptedIOException))
{
       throw new ManifoldCFException("Interrupted: " + e.getMessage(), e,
         ManifoldCFException.INTERRUPTED);
     }

Modified: manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java?rev=1489745&r1=1489744&r2=1489745&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java
(original)
+++ manifoldcf/trunk/connectors/googledrive/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/googledrive/GoogleDriveRepositoryConnector.java
Wed Jun  5 08:53:59 2013
@@ -720,6 +720,7 @@ public class GoogleDriveRepositoryConnec
     GetSeedsThread t = new GetSeedsThread(googleDriveQuery);
     try {
       t.start();
+      boolean wasInterrupted = false;
       try {
         XThreadStringBuffer seedBuffer = t.getBuffer();
         // Pick up the paths, and add them to the activities, before we join with the child
thread.
@@ -731,8 +732,16 @@ public class GoogleDriveRepositoryConnec
           // Add the pageID to the queue
           activities.addSeedDocument(docPath);
         }
+      } catch (InterruptedException e) {
+        wasInterrupted = true;
+        throw e;
+      } catch (ManifoldCFException e) {
+        if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+          wasInterrupted = true;
+        throw e;
       } finally {
-        t.finishUp();
+        if (!wasInterrupted)
+          t.finishUp();
       }
     } catch (InterruptedException e) {
       t.interrupt();
@@ -927,6 +936,7 @@ public class GoogleDriveRepositoryConnec
           GetChildrenThread t = new GetChildrenThread(nodeId);
           try {
             t.start();
+            boolean wasInterrupted = false;
             try {
               XThreadStringBuffer childBuffer = t.getBuffer();
               // Pick up the paths, and add them to the activities, before we join with the
child thread.
@@ -938,8 +948,16 @@ public class GoogleDriveRepositoryConnec
                 // Add the pageID to the queue
                 activities.addDocumentReference(child, nodeId, RELATIONSHIP_CHILD);
               }
+            } catch (InterruptedException e) {
+              wasInterrupted = true;
+              throw e;
+            } catch (ManifoldCFException e) {
+              if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+                wasInterrupted = true;
+              throw e;
             } finally {
-              t.finishUp();
+              if (!wasInterrupted)
+                t.finishUp();
             }
           } catch (InterruptedException e) {
             t.interrupt();
@@ -1002,6 +1020,7 @@ public class GoogleDriveRepositoryConnec
             DocumentReadingThread t = new DocumentReadingThread(documentURI);
             try {
               t.start();
+              boolean wasInterrupted = false;
               try {
                 InputStream is = t.getSafeInputStream();
                 try {
@@ -1011,8 +1030,18 @@ public class GoogleDriveRepositoryConnec
                 } finally {
                   is.close();
                 }
+              } catch (ManifoldCFException e) {
+                if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
+                  wasInterrupted = true;
+                throw e;
+              } catch (java.net.SocketTimeoutException e) {
+                throw e;
+              } catch (InterruptedIOException e) {
+                wasInterrupted = true;
+                throw e;
               } finally {
-                t.finishUp();
+                if (!wasInterrupted)
+                  t.finishUp();
               }
 
               // No errors.  Record the fact that we made it.
@@ -1196,9 +1225,12 @@ public class GoogleDriveRepositoryConnec
   
   private static void handleIOException(IOException e)
     throws ManifoldCFException, ServiceInterruption {
-    // MHL to deal with various kinds of IOException
+    if (!(e instanceof java.net.SocketTimeoutException) && (e instanceof InterruptedIOException))
{
+      throw new ManifoldCFException("Interrupted: " + e.getMessage(), e,
+        ManifoldCFException.INTERRUPTED);
+    }
     long currentTime = System.currentTimeMillis();
-    throw new ServiceInterruption("GoogleDrive exception: "+e.getMessage(), e, currentTime
+ 300000L,
+    throw new ServiceInterruption("IO exception: "+e.getMessage(), e, currentTime + 300000L,
       currentTime + 3 * 60 * 60000L,-1,false);
   }
   



Mime
View raw message