manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1433350 - in /manifoldcf/trunk: ./ connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/ connector...
Date Tue, 15 Jan 2013 11:01:59 GMT
Author: kwright
Date: Tue Jan 15 11:01:58 2013
New Revision: 1433350

URL: http://svn.apache.org/viewvc?rev=1433350&view=rev
Log:
Fix for CONNECTORS-612.

Modified:
    manifoldcf/trunk/CHANGES.txt
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
    manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
    manifoldcf/trunk/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/meridio/meridiowrapper/MeridioWrapper.java
    manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java
    manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
    manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
    manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
    manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
    manifoldcf/trunk/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java

Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Tue Jan 15 11:01:58 2013
@@ -3,6 +3,9 @@ $Id$
 
 ======================= 1.1-dev =====================
 
+CONNECTORS-612: Disable HttpClient retries everywhere.
+(Karl Wright)
+
 CONNECTORS-611: Handle inability to get permissions for funky docs
 by skipping those docs, rather than throwing an NPE.
 (Ahmet Arslan, Karl Wright)

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
Tue Jan 15 11:01:58 2013
@@ -30,6 +30,8 @@ import org.apache.http.conn.ClientConnec
 import org.apache.http.impl.conn.PoolingClientConnectionManager;
 import org.apache.http.client.HttpClient;
 import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.protocol.HttpContext;
 
 import org.apache.commons.io.FilenameUtils;
 import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
@@ -104,6 +106,18 @@ public class ElasticSearchConnector exte
     localConnectionManager.setMaxTotal(1);
     connectionManager = localConnectionManager;
     DefaultHttpClient localClient = new DefaultHttpClient(connectionManager);
+    // No retries
+    localClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
+      {
+	public boolean retryRequest(
+	  IOException exception,
+	  int executionCount,
+          HttpContext context)
+	{
+	  return false;
+	}
+     
+      });
     client = localClient;
   }
   

Modified: manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
(original)
+++ manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
Tue Jan 15 11:01:58 2013
@@ -64,6 +64,8 @@ import org.apache.http.client.params.Htt
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.protocol.HTTP;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.protocol.HttpContext;
 
 import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.client.RedirectException;
@@ -420,10 +422,23 @@ public class LivelinkConnector extends o
       params.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY,true);
       params.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,false);
       params.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
-      params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,60000);
+      params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,900000);
       params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,300000);
       params.setBooleanParameter(ClientPNames.HANDLE_REDIRECTS,true);
       DefaultHttpClient localHttpClient = new DefaultHttpClient(connectionManager,params);
+      // No retries
+      localHttpClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
+        {
+          public boolean retryRequest(
+            IOException exception,
+            int executionCount,
+            HttpContext context)
+          {
+            return false;
+          }
+       
+        });
+
       localHttpClient.setRedirectStrategy(new DefaultRedirectStrategy());
       // Set up authentication to use
       if (ntlmDomain != null)

Modified: manifoldcf/trunk/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/meridio/meridiowrapper/MeridioWrapper.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/meridio/meridiowrapper/MeridioWrapper.java?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/meridio/meridiowrapper/MeridioWrapper.java
(original)
+++ manifoldcf/trunk/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/meridio/meridiowrapper/MeridioWrapper.java
Tue Jan 15 11:01:58 2013
@@ -87,6 +87,8 @@ import org.apache.http.params.HttpParams
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.NTCredentials;
 import org.apache.http.HttpHost;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.protocol.HttpContext;
 
 
 public class MeridioWrapper
@@ -259,9 +261,22 @@ public class MeridioWrapper
     BasicHttpParams dmwsParams = new BasicHttpParams();
     dmwsParams.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY,true);
     dmwsParams.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,false);
-    dmwsParams.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,60000);
+    dmwsParams.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,900000);
     dmwsParams.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
     DefaultHttpClient localDmwsHttpClient = new DefaultHttpClient(connectionManager,dmwsParams);
+    // No retries
+    localDmwsHttpClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
+      {
+	public boolean retryRequest(
+	  IOException exception,
+	  int executionCount,
+          HttpContext context)
+	{
+	  return false;
+	}
+     
+      });
+
     localDmwsHttpClient.setRedirectStrategy(new DefaultRedirectStrategy());
     if (domainUser != null)
     {
@@ -290,9 +305,22 @@ public class MeridioWrapper
     BasicHttpParams rmwsParams = new BasicHttpParams();
     rmwsParams.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY,true);
     rmwsParams.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,false);
-    rmwsParams.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,60000);
+    rmwsParams.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,900000);
     rmwsParams.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
     DefaultHttpClient localRmwsHttpClient = new DefaultHttpClient(connectionManager,rmwsParams);
+    // No retries
+    localRmwsHttpClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
+      {
+	public boolean retryRequest(
+	  IOException exception,
+	  int executionCount,
+          HttpContext context)
+	{
+	  return false;
+	}
+     
+      });
+
     localRmwsHttpClient.setRedirectStrategy(new DefaultRedirectStrategy());
     if (domainUser != null)
     {
@@ -323,9 +351,22 @@ public class MeridioWrapper
       BasicHttpParams mcwsParams = new BasicHttpParams();
       mcwsParams.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY,true);
       mcwsParams.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,false);
-      mcwsParams.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,60000);
+      mcwsParams.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,900000);
       mcwsParams.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
       DefaultHttpClient localMcwsHttpClient = new DefaultHttpClient(connectionManager,mcwsParams);
+      // No retries
+      localMcwsHttpClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
+        {
+          public boolean retryRequest(
+            IOException exception,
+            int executionCount,
+            HttpContext context)
+          {
+            return false;
+          }
+       
+        });
+
       localMcwsHttpClient.setRedirectStrategy(new DefaultRedirectStrategy());
       if (domainUser != null)
       {

Modified: manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java
(original)
+++ manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java
Tue Jan 15 11:01:58 2013
@@ -53,6 +53,8 @@ import org.apache.http.params.HttpProtoc
 import org.apache.http.ProtocolVersion;
 import org.apache.http.util.EntityUtils;
 import org.apache.http.message.BasicHeader;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.protocol.HttpContext;
 
 import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.client.RedirectException;
@@ -111,6 +113,19 @@ public class OpenSearchServerConnection 
     localConnectionManager.setMaxTotal(1);
     connectionManager = localConnectionManager;
     DefaultHttpClient localHttpClient = new DefaultHttpClient(connectionManager);
+    // No retries
+    localHttpClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
+      {
+	public boolean retryRequest(
+	  IOException exception,
+	  int executionCount,
+          HttpContext context)
+	{
+	  return false;
+	}
+     
+      });
+
     httpClient = localHttpClient;
   }
 

Modified: manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
(original)
+++ manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
Tue Jan 15 11:01:58 2013
@@ -53,6 +53,8 @@ import org.apache.http.conn.params.ConnR
 import org.apache.http.message.BasicHeader;
 import org.apache.http.client.params.ClientPNames;
 import org.apache.http.conn.ssl.AllowAllHostnameVerifier;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.protocol.HttpContext;
 
 import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.client.CircularRedirectException;
@@ -287,6 +289,19 @@ public class ThrottledFetcher
       params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,connectionTimeoutMilliseconds);
       params.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
       DefaultHttpClient localHttpClient = new DefaultHttpClient(connectionManager,params);
+      // No retries
+      localHttpClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
+        {
+          public boolean retryRequest(
+            IOException exception,
+            int executionCount,
+            HttpContext context)
+          {
+            return false;
+          }
+         
+        });
+
       localHttpClient.setRedirectStrategy(new DefaultRedirectStrategy());
       
       // If there's a proxy, set that too.

Modified: manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
(original)
+++ manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
Tue Jan 15 11:01:58 2013
@@ -55,6 +55,8 @@ import org.apache.http.params.BasicHttpP
 import org.apache.http.params.HttpParams;
 import org.apache.http.params.CoreConnectionPNames;
 import org.apache.http.client.params.ClientPNames;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.protocol.HttpContext;
 
 /** This is the "repository connector" for Microsoft SharePoint.
 * Document identifiers for this connector come in three forms:
@@ -218,9 +220,21 @@ public class SharePointRepository extend
       BasicHttpParams params = new BasicHttpParams();
       params.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY,true);
       params.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,false);
-      params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,60000);
+      params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,900000);
       params.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
       DefaultHttpClient localHttpClient = new DefaultHttpClient(connectionManager,params);
+      // No retries
+      localHttpClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
+        {
+          public boolean retryRequest(
+            IOException exception,
+            int executionCount,
+            HttpContext context)
+          {
+            return false;
+          }
+       
+        });
       localHttpClient.setRedirectStrategy(new DefaultRedirectStrategy());
       if (strippedUserName != null)
       {

Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Tue Jan 15 11:01:58 2013
@@ -226,7 +226,7 @@ public class HttpPoster
 	public boolean retryRequest(
 	  IOException exception,
 	  int executionCount,
-	    HttpContext context)
+          HttpContext context)
 	{
 	  return false;
 	}

Modified: manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
(original)
+++ manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
Tue Jan 15 11:01:58 2013
@@ -77,6 +77,7 @@ import org.apache.http.protocol.HttpCont
 import org.apache.http.protocol.BasicHttpContext;
 import org.apache.http.client.protocol.ClientContext;
 import org.apache.http.cookie.CookieIdentityComparator;
+import org.apache.http.client.HttpRequestRetryHandler;
 
 import org.apache.http.cookie.MalformedCookieException;
 import org.apache.http.conn.ConnectTimeoutException;
@@ -1344,6 +1345,18 @@ public class ThrottledFetcher
         params.setBooleanParameter(CookieSpecPNames.SINGLE_COOKIE_HEADER,new Boolean(true));
 
         DefaultHttpClient localHttpClient = new DefaultHttpClient(connManager,params);
+        // No retries
+        localHttpClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
+          {
+            public boolean retryRequest(
+              IOException exception,
+              int executionCount,
+              HttpContext context)
+            {
+              return false;
+            }
+         
+          });
         localHttpClient.setRedirectStrategy(new DefaultRedirectStrategy());
         localHttpClient.getCookieSpecs().register(CookiePolicy.BROWSER_COMPATIBILITY, new
CookieSpecFactory()
           {

Modified: manifoldcf/trunk/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java?rev=1433350&r1=1433349&r2=1433350&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
(original)
+++ manifoldcf/trunk/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
Tue Jan 15 11:01:58 2013
@@ -50,6 +50,8 @@ import org.apache.http.protocol.HTTP;
 import org.apache.http.util.EntityUtils;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.client.params.ClientPNames;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.protocol.HttpContext;
 
 import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.client.CircularRedirectException;
@@ -173,10 +175,23 @@ public class WikiConnector extends org.a
       BasicHttpParams params = new BasicHttpParams();
       params.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY,true);
       params.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,false);
-      params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,300000);
+      params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,900000);
       params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,300000);
       params.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
       DefaultHttpClient localHttpClient = new DefaultHttpClient(connectionManager,params);
+      // No retries
+      localHttpClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
+        {
+          public boolean retryRequest(
+            IOException exception,
+            int executionCount,
+            HttpContext context)
+          {
+            return false;
+          }
+       
+        });
+
       httpClient = localHttpClient;
       
       loginToAPI();



Mime
View raw message