manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1432669 - in /manifoldcf/trunk: CHANGES.txt connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
Date Sun, 13 Jan 2013 17:34:15 GMT
Author: kwright
Date: Sun Jan 13 17:34:12 2013
New Revision: 1432669

URL: http://svn.apache.org/viewvc?rev=1432669&view=rev
Log:
Fix for CONNECTORS-608.  Increase default socket timeout to 15 minutes.  Also change handling
of socket timeouts to skip the document rather than abort the job.

Modified:
    manifoldcf/trunk/CHANGES.txt
    manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
    manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java

Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1432669&r1=1432668&r2=1432669&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Sun Jan 13 17:34:12 2013
@@ -3,6 +3,10 @@ $Id$
 
 ======================= 1.1-dev =====================
 
+CONNECTORS-608: Solr connector times out chronically on some
+documents.
+(David Morana, Karl Wright)
+
 CONNECTORS-607: Handle multi-valued parameters properly in the
 Solr connector.
 (Minoru Osuka, Shinichiro Abe, Karl Wright)

Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1432669&r1=1432668&r2=1432669&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Sun Jan 13 17:34:12 2013
@@ -370,19 +370,31 @@ public class HttpPoster
     if ((e instanceof InterruptedIOException) && (!(e instanceof java.net.SocketTimeoutException)))
       throw new ManifoldCFException(e.getMessage(), ManifoldCFException.INTERRUPTED);
 
-    // Intercept "broken pipe" exception, since that seems to be what we get if the ingestion
API kills the socket right after a 400 goes out.
-    // Basically, we have no choice but to interpret that in the same manner as a 400, since
no matter how we do it, it's a race and the 'broken pipe'
-    // result is always possible.  So we might as well expect it and treat it properly.
-    if (e.getClass().getName().equals("java.net.SocketException") && e.getMessage().toLowerCase().indexOf("broken
pipe") != -1)
-    {
-      // We've seen what looks like the ingestion interface forcibly closing the socket.
-      // We *choose* to interpret this just like a 400 response.  However, we log in the
history using a different code,
-      // since we really don't know what happened for sure.
-      return;
+    long currentTime = System.currentTimeMillis();
+    
+    if (e.getClass().getName().equals("java.net.SocketException"))
+    {
+      // Intercept "broken pipe" exception, since that seems to be what we get if the ingestion
API kills the socket right after a 400 goes out.
+      // Basically, we have no choice but to interpret that in the same manner as a 400,
since no matter how we do it, it's a race and the 'broken pipe'
+      // result is always possible.  So we might as well expect it and treat it properly.
+      if (e.getMessage().toLowerCase().indexOf("broken pipe") != -1)
+        // We've seen what looks like the ingestion interface forcibly closing the socket.
+        // We *choose* to interpret this just like a 400 response.  However, we log in the
history using a different code,
+        // since we really don't know what happened for sure.
+        return;
+      
+      // Other socket exceptions are service interruptions - but if we keep getting them,
it means 
+      // that a socket timeout is probably set too low to accept this particular document.
 So
+      // we retry for a while, then skip the document.
+      throw new ServiceInterruption("Socket timeout exception during "+context+": "+e.getMessage(),
+        e,
+        currentTime + interruptionRetryTime,
+        currentTime + 1L * 60L * 60000L,
+        -1,
+        false);
     }
     
-    // Otherwise, presume that retries might fix it.
-    long currentTime = System.currentTimeMillis();
+    // Otherwise, no idea what the trouble is, so presume that retries might fix it.
     throw new ServiceInterruption("IO exception during "+context+": "+e.getMessage(),
       e,
       currentTime + interruptionRetryTime,

Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java?rev=1432669&r1=1432668&r2=1432669&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
(original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
Sun Jan 13 17:34:12 2013
@@ -226,7 +226,7 @@ public class SolrConnector extends org.a
         // Pick up timeouts
         String socketTimeoutString = params.getParameter(SolrConfig.PARAM_SOCKET_TIMEOUT);
         if (socketTimeoutString == null)
-          socketTimeoutString = "60";
+          socketTimeoutString = "900";
         String connectTimeoutString = params.getParameter(SolrConfig.PARAM_CONNECTION_TIMEOUT);
         if (connectTimeoutString == null)
           connectTimeoutString = "60";
@@ -967,7 +967,7 @@ public class SolrConnector extends org.a
     
     String socketTimeout = parameters.getParameter(SolrConfig.PARAM_SOCKET_TIMEOUT);
     if (socketTimeout == null)
-      socketTimeout = "60";
+      socketTimeout = "900";
 
     String zkClientTimeout = parameters.getParameter(SolrConfig.PARAM_ZOOKEEPER_CLIENT_TIMEOUT);
     if (zkClientTimeout == null)



Mime
View raw message