manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1672616 - in /manifoldcf/trunk: ./ connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/ connectors/j...
Date Fri, 10 Apr 2015 11:47:50 GMT
Author: kwright
Date: Fri Apr 10 11:47:49 2015
New Revision: 1672616

URL: http://svn.apache.org/r1672616
Log:
Fix for CONNECTORS-1185.

Added:
    manifoldcf/trunk/framework/connector-common/src/main/java/org/apache/http/
    manifoldcf/trunk/framework/connector-common/src/main/java/org/apache/http/impl/
    manifoldcf/trunk/framework/connector-common/src/main/java/org/apache/http/impl/cookie/
    manifoldcf/trunk/framework/connector-common/src/main/java/org/apache/http/impl/cookie/LaxBrowserCompatSpec.java   (with props)
Modified:
    manifoldcf/trunk/CHANGES.txt
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
    manifoldcf/trunk/connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/JiraSession.java
    manifoldcf/trunk/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraSession.java
    manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
    manifoldcf/trunk/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/meridio/MeridioWrapper.java
    manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
    manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
    manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointAuthority.java
    manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
    manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
    manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IThrottledConnection.java
    manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
    manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java
    manifoldcf/trunk/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
    manifoldcf/trunk/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyShutdown.java
    manifoldcf/trunk/framework/script-engine/src/main/java/org/apache/manifoldcf/scriptengine/ScriptParser.java

Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1672616&r1=1672615&r2=1672616&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Fri Apr 10 11:47:49 2015
@@ -3,6 +3,10 @@ $Id$
 
 ======================= 2.1-dev =====================
 
+CONNECTORS-1185: Fix HttpClient deprecation warnings, and make sure
+socket timeouts are actually honored in http connections.
+(Karl Wright)
+
 CONNECTORS-1183: Fix for GridFS connector broken UI server tab.
 (Kishore Kumar)
 

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1672616&r1=1672615&r2=1672616&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java Fri Apr 10 11:47:49 2015
@@ -113,20 +113,24 @@ public class ElasticSearchConnector exte
   {
     if (client == null)
     {
-      connectionManager = new PoolingHttpClientConnectionManager();
-      
       int socketTimeout = 900000;
       int connectionTimeout = 60000;
       
       // Set up connection manager
-      connectionManager = new PoolingHttpClientConnectionManager();
+      PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+      poolingConnectionManager.setDefaultMaxPerRoute(1);
+      poolingConnectionManager.setValidateAfterInactivity(60000);
+      poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+        .setTcpNoDelay(true)
+        .setSoTimeout(socketTimeout)
+        .build());
+      connectionManager = poolingConnectionManager;
 
       CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
 
       RequestConfig.Builder requestBuilder = RequestConfig.custom()
           .setCircularRedirectsAllowed(true)
           .setSocketTimeout(socketTimeout)
-          .setStaleConnectionCheckEnabled(true)
           .setExpectContinueEnabled(true)
           .setConnectTimeout(connectionTimeout)
           .setConnectionRequestTimeout(socketTimeout);
@@ -136,10 +140,6 @@ public class ElasticSearchConnector exte
         .setMaxConnTotal(1)
         .disableAutomaticRetries()
         .setDefaultRequestConfig(requestBuilder.build())
-        .setDefaultSocketConfig(SocketConfig.custom()
-          //.setTcpNoDelay(true)
-          .setSoTimeout(socketTimeout)
-          .build())
         .setDefaultCredentialsProvider(credentialsProvider)
         .setRequestExecutor(new HttpRequestExecutor(socketTimeout))
         .build();

Modified: manifoldcf/trunk/connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/JiraSession.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/JiraSession.java?rev=1672616&r1=1672615&r2=1672616&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/JiraSession.java (original)
+++ manifoldcf/trunk/connectors/jira/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/jira/JiraSession.java Fri Apr 10 11:47:49 2015
@@ -64,6 +64,7 @@ import org.apache.http.util.EntityUtils;
 import org.apache.http.client.HttpRequestRetryHandler;
 import org.apache.http.protocol.HttpContext;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.entity.ContentType;
 import org.apache.http.client.AuthCache;
 import org.apache.http.impl.client.BasicAuthCache;
@@ -124,9 +125,16 @@ public class JiraSession {
 
     javax.net.ssl.SSLSocketFactory httpsSocketFactory = KeystoreManagerFactory.getTrustingSecureSocketFactory();
     SSLConnectionSocketFactory myFactory = new SSLConnectionSocketFactory(new InterruptibleSocketFactory(httpsSocketFactory,connectionTimeout),
-      SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+      NoopHostnameVerifier.INSTANCE);
 
-    connectionManager = new PoolingHttpClientConnectionManager();
+    PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+    poolingConnectionManager.setDefaultMaxPerRoute(1);
+    poolingConnectionManager.setValidateAfterInactivity(60000);
+    poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+      .setTcpNoDelay(true)
+      .setSoTimeout(socketTimeout)
+      .build());
+    connectionManager = poolingConnectionManager;
 
     CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
 
@@ -141,7 +149,6 @@ public class JiraSession {
     RequestConfig.Builder requestBuilder = RequestConfig.custom()
       .setCircularRedirectsAllowed(true)
       .setSocketTimeout(socketTimeout)
-      .setStaleConnectionCheckEnabled(true)
       .setExpectContinueEnabled(true)
       .setConnectTimeout(connectionTimeout)
       .setConnectionRequestTimeout(socketTimeout);
@@ -169,13 +176,8 @@ public class JiraSession {
 
     httpClient = HttpClients.custom()
       .setConnectionManager(connectionManager)
-      .setMaxConnTotal(1)
       .disableAutomaticRetries()
       .setDefaultRequestConfig(requestBuilder.build())
-      .setDefaultSocketConfig(SocketConfig.custom()
-        .setTcpNoDelay(true)
-        .setSoTimeout(socketTimeout)
-        .build())
       .setDefaultCredentialsProvider(credentialsProvider)
       .setSSLSocketFactory(myFactory)
       .setRequestExecutor(new HttpRequestExecutor(socketTimeout))

Modified: manifoldcf/trunk/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraSession.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraSession.java?rev=1672616&r1=1672615&r2=1672616&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraSession.java (original)
+++ manifoldcf/trunk/connectors/jira/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/jira/JiraSession.java Fri Apr 10 11:47:49 2015
@@ -62,6 +62,7 @@ import org.apache.http.util.EntityUtils;
 import org.apache.http.client.HttpRequestRetryHandler;
 import org.apache.http.protocol.HttpContext;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.entity.ContentType;
 import org.apache.http.client.AuthCache;
 import org.apache.http.impl.client.BasicAuthCache;
@@ -121,9 +122,16 @@ public class JiraSession {
 
     javax.net.ssl.SSLSocketFactory httpsSocketFactory = KeystoreManagerFactory.getTrustingSecureSocketFactory();
     SSLConnectionSocketFactory myFactory = new SSLConnectionSocketFactory(new InterruptibleSocketFactory(httpsSocketFactory,connectionTimeout),
-      SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+      NoopHostnameVerifier.INSTANCE);
 
-    connectionManager = new PoolingHttpClientConnectionManager();
+    PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+    poolingConnectionManager.setDefaultMaxPerRoute(1);
+    poolingConnectionManager.setValidateAfterInactivity(60000);
+    poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+      .setTcpNoDelay(true)
+      .setSoTimeout(socketTimeout)
+      .build());
+    connectionManager = poolingConnectionManager;
 
     CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
 
@@ -138,7 +146,6 @@ public class JiraSession {
     RequestConfig.Builder requestBuilder = RequestConfig.custom()
       .setCircularRedirectsAllowed(true)
       .setSocketTimeout(socketTimeout)
-      .setStaleConnectionCheckEnabled(true)
       .setExpectContinueEnabled(true)
       .setConnectTimeout(connectionTimeout)
       .setConnectionRequestTimeout(socketTimeout);
@@ -167,13 +174,8 @@ public class JiraSession {
 
     httpClient = HttpClients.custom()
       .setConnectionManager(connectionManager)
-      .setMaxConnTotal(1)
       .disableAutomaticRetries()
       .setDefaultRequestConfig(requestBuilder.build())
-      .setDefaultSocketConfig(SocketConfig.custom()
-        .setTcpNoDelay(true)
-        .setSoTimeout(socketTimeout)
-        .build())
       .setDefaultCredentialsProvider(credentialsProvider)
       .setSSLSocketFactory(myFactory)
       .setRequestExecutor(new HttpRequestExecutor(socketTimeout))

Modified: manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java?rev=1672616&r1=1672615&r2=1672616&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java (original)
+++ manifoldcf/trunk/connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/LivelinkConnector.java Fri Apr 10 11:47:49 2015
@@ -53,7 +53,7 @@ import org.apache.http.client.methods.Ht
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.NameValuePair;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.auth.AuthScope;
@@ -525,8 +525,15 @@ public class LivelinkConnector extends o
       int connectionTimeout = 300000;
 
       // Set up connection manager
-      connectionManager = new PoolingHttpClientConnectionManager();
-
+      PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+      poolingConnectionManager.setDefaultMaxPerRoute(1);
+      poolingConnectionManager.setValidateAfterInactivity(60000);
+      poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+        .setTcpNoDelay(true)
+        .setSoTimeout(socketTimeout)
+        .build());
+      connectionManager = poolingConnectionManager;
+      
       CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
 
       // Set up ingest ssl if indicated
@@ -534,7 +541,7 @@ public class LivelinkConnector extends o
       if (ingestKeystoreManager != null)
       {
         myFactory = new SSLConnectionSocketFactory(new InterruptibleSocketFactory(ingestKeystoreManager.getSecureSocketFactory(), connectionTimeout),
-          new BrowserCompatHostnameVerifier());
+          NoopHostnameVerifier.INSTANCE);
       }
 
       // Set up authentication to use
@@ -546,20 +553,14 @@ public class LivelinkConnector extends o
 
       HttpClientBuilder builder = HttpClients.custom()
         .setConnectionManager(connectionManager)
-        .setMaxConnTotal(1)
         .disableAutomaticRetries()
         .setDefaultRequestConfig(RequestConfig.custom()
           .setCircularRedirectsAllowed(true)
           .setSocketTimeout(socketTimeout)
-          .setStaleConnectionCheckEnabled(true)
           .setExpectContinueEnabled(true)
           .setConnectTimeout(connectionTimeout)
           .setConnectionRequestTimeout(socketTimeout)
           .build())
-        .setDefaultSocketConfig(SocketConfig.custom()
-          .setTcpNoDelay(true)
-          .setSoTimeout(socketTimeout)
-          .build())
         .setDefaultCredentialsProvider(credentialsProvider)
         .setRequestExecutor(new HttpRequestExecutor(socketTimeout))
         .setRedirectStrategy(new DefaultRedirectStrategy());

Modified: manifoldcf/trunk/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/meridio/MeridioWrapper.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/meridio/MeridioWrapper.java?rev=1672616&r1=1672615&r2=1672616&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/meridio/MeridioWrapper.java (original)
+++ manifoldcf/trunk/connectors/meridio/connector/src/main/java/org/apache/manifoldcf/meridio/MeridioWrapper.java Fri Apr 10 11:47:49 2015
@@ -82,7 +82,7 @@ import org.apache.http.client.Credential
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.config.SocketConfig;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.impl.client.DefaultRedirectStrategy;
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.NTCredentials;
@@ -218,7 +218,7 @@ public class MeridioWrapper
     SSLConnectionSocketFactory myFactory = null;
     if (mySSLFactory != null)
     {
-      myFactory = new SSLConnectionSocketFactory(mySSLFactory, new BrowserCompatHostnameVerifier());
+      myFactory = new SSLConnectionSocketFactory(mySSLFactory, new NoopHostnameVerifier());
     }
 
     // Set up the pool.
@@ -257,9 +257,32 @@ public class MeridioWrapper
     int socketTimeout = 900000;
     int connectionTimeout = 300000;
 
-    dmwsConnectionManager = new PoolingHttpClientConnectionManager();
-    rmwsConnectionManager = new PoolingHttpClientConnectionManager();
-    mcwsConnectionManager = new PoolingHttpClientConnectionManager();
+    PoolingHttpClientConnectionManager poolingDmwsConnectionManager = new PoolingHttpClientConnectionManager();
+    poolingDmwsConnectionManager.setDefaultMaxPerRoute(1);
+    poolingDmwsConnectionManager.setValidateAfterInactivity(60000);
+    poolingDmwsConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+      .setTcpNoDelay(true)
+      .setSoTimeout(socketTimeout)
+      .build());
+    dmwsConnectionManager = poolingDmwsConnectionManager;
+    
+    PoolingHttpClientConnectionManager poolingRmwsConnectionManager = new PoolingHttpClientConnectionManager();
+    poolingRmwsConnectionManager.setDefaultMaxPerRoute(1);
+    poolingRmwsConnectionManager.setValidateAfterInactivity(60000);
+    poolingRmwsConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+      .setTcpNoDelay(true)
+      .setSoTimeout(socketTimeout)
+      .build());
+    rmwsConnectionManager = poolingRmwsConnectionManager;
+    
+    PoolingHttpClientConnectionManager poolingMcwsConnectionManager = new PoolingHttpClientConnectionManager();
+    poolingMcwsConnectionManager.setDefaultMaxPerRoute(1);
+    poolingMcwsConnectionManager.setValidateAfterInactivity(60000);
+    poolingMcwsConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+      .setTcpNoDelay(true)
+      .setSoTimeout(socketTimeout)
+      .build());
+    mcwsConnectionManager = poolingMcwsConnectionManager;
 
     // Initialize the three httpclient objects
 
@@ -269,14 +292,12 @@ public class MeridioWrapper
     RequestConfig.Builder dmwsRequestBuilder = RequestConfig.custom()
       .setCircularRedirectsAllowed(true)
       .setSocketTimeout(socketTimeout)
-      .setStaleConnectionCheckEnabled(true)
       .setExpectContinueEnabled(false)
       .setConnectTimeout(connectionTimeout)
       .setConnectionRequestTimeout(socketTimeout);
     RequestConfig.Builder rmwsRequestBuilder = RequestConfig.custom()
       .setCircularRedirectsAllowed(true)
       .setSocketTimeout(socketTimeout)
-      .setStaleConnectionCheckEnabled(true)
       .setExpectContinueEnabled(true)
       .setConnectTimeout(connectionTimeout)
       .setConnectionRequestTimeout(socketTimeout);
@@ -326,13 +347,8 @@ public class MeridioWrapper
 
     dmwsHttpClient = HttpClients.custom()
       .setConnectionManager(dmwsConnectionManager)
-      .setMaxConnTotal(1)
       .disableAutomaticRetries()
       .setDefaultRequestConfig(dmwsRequestBuilder.build())
-      .setDefaultSocketConfig(SocketConfig.custom()
-        .setTcpNoDelay(true)
-        .setSoTimeout(socketTimeout)
-        .build())
       .setDefaultCredentialsProvider(dmwsCredentialsProvider)
       .setSSLSocketFactory(myFactory)
       .setRequestExecutor(new HttpRequestExecutor(socketTimeout))
@@ -341,13 +357,8 @@ public class MeridioWrapper
     
     rmwsHttpClient = HttpClients.custom()
       .setConnectionManager(rmwsConnectionManager)
-      .setMaxConnTotal(1)
       .disableAutomaticRetries()
       .setDefaultRequestConfig(rmwsRequestBuilder.build())
-      .setDefaultSocketConfig(SocketConfig.custom()
-        .setTcpNoDelay(true)
-        .setSoTimeout(socketTimeout)
-        .build())
       .setDefaultCredentialsProvider(rmwsCredentialsProvider)
       .setSSLSocketFactory(myFactory)
       .setRequestExecutor(new HttpRequestExecutor(socketTimeout))
@@ -361,7 +372,6 @@ public class MeridioWrapper
       RequestConfig.Builder mcwsRequestBuilder = RequestConfig.custom()
         .setCircularRedirectsAllowed(true)
         .setSocketTimeout(socketTimeout)
-        .setStaleConnectionCheckEnabled(true)
         .setExpectContinueEnabled(true)
         .setConnectTimeout(connectionTimeout)
         .setConnectionRequestTimeout(socketTimeout);
@@ -391,13 +401,8 @@ public class MeridioWrapper
       
       mcwsHttpClient = HttpClients.custom()
         .setConnectionManager(mcwsConnectionManager)
-        .setMaxConnTotal(1)
         .disableAutomaticRetries()
         .setDefaultRequestConfig(mcwsRequestBuilder.build())
-        .setDefaultSocketConfig(SocketConfig.custom()
-          .setTcpNoDelay(true)
-          .setSoTimeout(socketTimeout)
-          .build())
         .setDefaultCredentialsProvider(mcwsCredentialsProvider)
         .setSSLSocketFactory(myFactory)
         .setRequestExecutor(new HttpRequestExecutor(socketTimeout))

Modified: manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java?rev=1672616&r1=1672615&r2=1672616&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java (original)
+++ manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java Fri Apr 10 11:47:49 2015
@@ -104,28 +104,31 @@ public class OpenSearchServerConnector e
   {
     if (client == null)
     {
-      connectionManager = new PoolingHttpClientConnectionManager();
-
       final int executorTimeout = 300000;
       final int socketTimeout = 60000;
       final int connectionTimeout = 60000;
 
+      PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+      poolingConnectionManager.setDefaultMaxPerRoute(1);
+      poolingConnectionManager.setValidateAfterInactivity(60000);
+      poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+        .setTcpNoDelay(true)
+        .setSoTimeout(socketTimeout)
+        .build());
+      connectionManager = poolingConnectionManager;
+
       RequestConfig.Builder requestBuilder = RequestConfig.custom()
           .setCircularRedirectsAllowed(true).setSocketTimeout(socketTimeout)
-          .setStaleConnectionCheckEnabled(true).setExpectContinueEnabled(true)
+          .setExpectContinueEnabled(true)
           .setConnectTimeout(connectionTimeout)
           .setConnectionRequestTimeout(socketTimeout);
 
       HttpClientBuilder clientBuilder = HttpClients
           .custom()
           .setConnectionManager(connectionManager)
-          .setMaxConnTotal(1)
           .disableAutomaticRetries()
           .setDefaultRequestConfig(requestBuilder.build())
-          .setRequestExecutor(new HttpRequestExecutor(executorTimeout))
-          .setDefaultSocketConfig(
-              SocketConfig.custom().setTcpNoDelay(true)
-                  .setSoTimeout(socketTimeout).build());
+          .setRequestExecutor(new HttpRequestExecutor(executorTimeout));
 
       client = clientBuilder.build();
 

Modified: manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java?rev=1672616&r1=1672615&r2=1672616&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java (original)
+++ manifoldcf/trunk/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java Fri Apr 10 11:47:49 2015
@@ -40,7 +40,7 @@ import org.apache.http.client.config.Req
 import org.apache.http.client.CredentialsProvider;
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.config.SocketConfig;
-import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
@@ -261,16 +261,22 @@ public class ThrottledFetcher
       // Create the https scheme for this connection
       javax.net.ssl.SSLSocketFactory httpsSocketFactory = KeystoreManagerFactory.getTrustingSecureSocketFactory();;
       SSLConnectionSocketFactory myFactory = new SSLConnectionSocketFactory(new InterruptibleSocketFactory(httpsSocketFactory,connectionTimeoutMilliseconds),
-        SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+        NoopHostnameVerifier.INSTANCE);
 
-      connectionManager = new PoolingHttpClientConnectionManager();
+      PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+      poolingConnectionManager.setDefaultMaxPerRoute(1);
+      poolingConnectionManager.setValidateAfterInactivity(60000);
+      poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+        .setTcpNoDelay(true)
+        .setSoTimeout(connectionTimeoutMilliseconds)
+        .build());
+      connectionManager = poolingConnectionManager;
 
       CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
 
       RequestConfig.Builder requestBuilder = RequestConfig.custom()
           .setCircularRedirectsAllowed(true)
           .setSocketTimeout(connectionTimeoutMilliseconds)
-          .setStaleConnectionCheckEnabled(true)
           .setExpectContinueEnabled(true)
           .setConnectTimeout(connectionTimeoutMilliseconds)
           .setConnectionRequestTimeout(connectionTimeoutMilliseconds);
@@ -302,10 +308,6 @@ public class ThrottledFetcher
         .setMaxConnTotal(1)
         .disableAutomaticRetries()
         .setDefaultRequestConfig(requestBuilder.build())
-        .setDefaultSocketConfig(SocketConfig.custom()
-          .setTcpNoDelay(true)
-          .setSoTimeout(connectionTimeoutMilliseconds)
-          .build())
         .setDefaultCredentialsProvider(credentialsProvider)
         .setSSLSocketFactory(myFactory)
         .setRequestExecutor(new HttpRequestExecutor(connectionTimeoutMilliseconds))

Modified: manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointAuthority.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointAuthority.java?rev=1672616&r1=1672615&r2=1672616&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointAuthority.java (original)
+++ manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/authorities/authorities/sharepoint/SharePointAuthority.java Fri Apr 10 11:47:49 2015
@@ -46,7 +46,7 @@ import org.apache.http.client.Credential
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.config.SocketConfig;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
+import org.apache.http.conn.ssl.DefaultHostnameVerifier;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.auth.AuthScope;
@@ -792,7 +792,14 @@ public class SharePointAuthority extends
       
       // Set up ssl if indicated
 
-      connectionManager = new PoolingHttpClientConnectionManager();
+      PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+      poolingConnectionManager.setDefaultMaxPerRoute(1);
+      poolingConnectionManager.setValidateAfterInactivity(60000);
+      poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+        .setTcpNoDelay(true)
+        .setSoTimeout(socketTimeout)
+        .build());
+      connectionManager = poolingConnectionManager;
 
       CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
 
@@ -800,7 +807,7 @@ public class SharePointAuthority extends
       if (keystoreData != null)
       {
         keystoreManager = KeystoreManagerFactory.make("",keystoreData);
-        myFactory = new SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(), new BrowserCompatHostnameVerifier());
+        myFactory = new SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(), new DefaultHostnameVerifier());
       }
 
       if (strippedUserName != null)
@@ -813,7 +820,6 @@ public class SharePointAuthority extends
       RequestConfig.Builder requestBuilder = RequestConfig.custom()
           .setCircularRedirectsAllowed(true)
           .setSocketTimeout(socketTimeout)
-          .setStaleConnectionCheckEnabled(true)
           .setExpectContinueEnabled(false)
           .setConnectTimeout(connectionTimeout)
           .setConnectionRequestTimeout(socketTimeout);
@@ -842,13 +848,8 @@ public class SharePointAuthority extends
 
       HttpClientBuilder builder = HttpClients.custom()
         .setConnectionManager(connectionManager)
-        .setMaxConnTotal(1)
         .disableAutomaticRetries()
         .setDefaultRequestConfig(requestBuilder.build())
-        .setDefaultSocketConfig(SocketConfig.custom()
-          .setTcpNoDelay(true)
-          .setSoTimeout(socketTimeout)
-          .build())
         .setDefaultCredentialsProvider(credentialsProvider);
       if (myFactory != null)
         builder.setSSLSocketFactory(myFactory);

Modified: manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java?rev=1672616&r1=1672615&r2=1672616&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java (original)
+++ manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java Fri Apr 10 11:47:49 2015
@@ -56,7 +56,7 @@ import org.apache.http.client.Credential
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.config.SocketConfig;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
+import org.apache.http.conn.ssl.DefaultHostnameVerifier;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.auth.AuthScope;
@@ -255,15 +255,22 @@ public class SharePointRepository extend
       int connectionTimeout = 60000;
       int socketTimeout = 900000;
 
-      connectionManager = new PoolingHttpClientConnectionManager();
-
+      PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+      poolingConnectionManager.setDefaultMaxPerRoute(1);
+      poolingConnectionManager.setValidateAfterInactivity(60000);
+      poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+        .setTcpNoDelay(true)
+        .setSoTimeout(socketTimeout)
+        .build());
+      connectionManager = poolingConnectionManager;
+      
       CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
 
       SSLConnectionSocketFactory myFactory = null;
       if (keystoreData != null)
       {
         keystoreManager = KeystoreManagerFactory.make("",keystoreData);
-        myFactory = new SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(), new BrowserCompatHostnameVerifier());
+        myFactory = new SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(), new DefaultHostnameVerifier());
       }
 
       if (strippedUserName != null)
@@ -276,7 +283,6 @@ public class SharePointRepository extend
       RequestConfig.Builder requestBuilder = RequestConfig.custom()
           .setCircularRedirectsAllowed(true)
           .setSocketTimeout(socketTimeout)
-          .setStaleConnectionCheckEnabled(true)
           .setExpectContinueEnabled(false)
           .setConnectTimeout(connectionTimeout)
           .setConnectionRequestTimeout(socketTimeout);
@@ -305,13 +311,8 @@ public class SharePointRepository extend
 
       HttpClientBuilder builder = HttpClients.custom()
         .setConnectionManager(connectionManager)
-        .setMaxConnTotal(1)
         .disableAutomaticRetries()
         .setDefaultRequestConfig(requestBuilder.build())
-        .setDefaultSocketConfig(SocketConfig.custom()
-          .setTcpNoDelay(true)
-          .setSoTimeout(socketTimeout)
-          .build())
         .setDefaultCredentialsProvider(credentialsProvider);
       if (myFactory != null)
         builder.setSSLSocketFactory(myFactory);

Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1672616&r1=1672615&r2=1672616&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java Fri Apr 10 11:47:49 2015
@@ -23,6 +23,7 @@ import org.apache.http.client.HttpClient
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.config.SocketConfig;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.impl.client.*;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.protocol.HttpRequestExecutor;
@@ -212,40 +213,39 @@ public class HttpPoster
     // Initialize standard solr-j.
     // First, we need an HttpClient where basic auth is properly set up.
     connectionManager = new PoolingHttpClientConnectionManager();
-    connectionManager.setMaxTotal(1);
-
+    connectionManager.setDefaultMaxPerRoute(1);
+    connectionManager.setValidateAfterInactivity(60000);
+    connectionManager.setDefaultSocketConfig(SocketConfig.custom()
+      .setTcpNoDelay(true)
+      .setSoTimeout(socketTimeout)
+      .build());
+    
     SSLConnectionSocketFactory myFactory;
     if (keystoreManager != null)
     {
-      myFactory = new SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(), SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+      myFactory = new SSLConnectionSocketFactory(keystoreManager.getSecureSocketFactory(), NoopHostnameVerifier.INSTANCE);
     }
     else
     {
       // Use the "trust everything" one
-      myFactory = new SSLConnectionSocketFactory(KeystoreManagerFactory.getTrustingSecureSocketFactory(),SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+      myFactory = new SSLConnectionSocketFactory(KeystoreManagerFactory.getTrustingSecureSocketFactory(),NoopHostnameVerifier.INSTANCE);
     }
 
     RequestConfig.Builder requestBuilder = RequestConfig.custom()
       .setCircularRedirectsAllowed(true)
       .setSocketTimeout(socketTimeout)
-      .setStaleConnectionCheckEnabled(true)
       .setExpectContinueEnabled(true)
       .setConnectTimeout(connectionTimeout)
       .setConnectionRequestTimeout(socketTimeout);
 
-      HttpClientBuilder clientBuilder = HttpClients.custom()
-        .setConnectionManager(connectionManager)
-        .setMaxConnTotal(1)
-        .disableAutomaticRetries()
-        .setDefaultRequestConfig(requestBuilder.build())
-        .setRedirectStrategy(new DefaultRedirectStrategy())
-        .setSSLSocketFactory(myFactory)
-        .setRequestExecutor(new HttpRequestExecutor(socketTimeout))
-        .setDefaultSocketConfig(SocketConfig.custom()
-          .setTcpNoDelay(true)
-          .setSoTimeout(socketTimeout)
-          .build()
-        );
+    HttpClientBuilder clientBuilder = HttpClients.custom()
+      .setConnectionManager(connectionManager)
+      .setMaxConnTotal(1)
+      .disableAutomaticRetries()
+      .setDefaultRequestConfig(requestBuilder.build())
+      .setRedirectStrategy(new DefaultRedirectStrategy())
+      .setSSLSocketFactory(myFactory)
+      .setRequestExecutor(new HttpRequestExecutor(socketTimeout));
 
 
     if (userID != null && userID.length() > 0 && password != null)

Modified: manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IThrottledConnection.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IThrottledConnection.java?rev=1672616&r1=1672615&r2=1672616&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IThrottledConnection.java (original)
+++ manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IThrottledConnection.java Fri Apr 10 11:47:49 2015
@@ -66,14 +66,13 @@ public interface IThrottledConnection
   * @param urlPath is the path part of the url, e.g. "/robots.txt"
   * @param userAgent is the value of the userAgent header to use.
   * @param from is the value of the from header to use.
-  * @param connectionTimeoutMilliseconds is the maximum number of milliseconds to wait on socket connect.
   * @param redirectOK should be set to true if you want redirects to be automatically followed.
   * @param host is the value to use as the "Host" header, or null to use the default.
   * @param formData describes additional form arguments and how to fetch the page.
   * @param loginCookies describes the cookies that should be in effect for this page fetch.
   */
-  public void executeFetch(String urlPath, String userAgent, String from, int connectionTimeoutMilliseconds,
-    int socketTimeoutMilliseconds, boolean redirectOK, String host, FormData formData,
+  public void executeFetch(String urlPath, String userAgent, String from,
+    boolean redirectOK, String host, FormData formData,
     LoginCookies loginCookies)
     throws ManifoldCFException, ServiceInterruption;
 

Modified: manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java?rev=1672616&r1=1672615&r2=1672616&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java (original)
+++ manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java Fri Apr 10 11:47:49 2015
@@ -49,7 +49,7 @@ import org.apache.http.client.methods.Ht
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.NameValuePair;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.auth.AuthScope;
@@ -69,9 +69,16 @@ import org.apache.http.protocol.HTTP;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.cookie.CookieOrigin;
 import org.apache.http.cookie.ClientCookie;
+import org.apache.http.cookie.CookieSpecProvider;
 import org.apache.http.cookie.Cookie;
+import org.apache.http.impl.cookie.DefaultCookieSpec;
+import org.apache.http.impl.cookie.LaxBrowserCompatSpec;
+import org.apache.http.impl.cookie.RFC6265CookieSpecProvider;
 import org.apache.http.impl.cookie.BasicPathHandler;
-import org.apache.http.impl.cookie.BrowserCompatSpec;
+import org.apache.http.impl.cookie.LaxMaxAgeHandler;
+import org.apache.http.impl.cookie.BasicDomainHandler;
+import org.apache.http.impl.cookie.BasicSecureHandler;
+import org.apache.http.impl.cookie.LaxExpiresHandler;
 import org.apache.http.cookie.CookieSpec;
 import org.apache.http.client.CookieStore;
 import org.apache.http.protocol.HttpContext;
@@ -79,15 +86,9 @@ import org.apache.http.protocol.BasicHtt
 import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.cookie.CookieIdentityComparator;
 import org.apache.http.client.HttpRequestRetryHandler;
-import org.apache.http.cookie.CookieSpecProvider;
 import org.apache.http.config.RegistryBuilder;
 import org.apache.http.config.Registry;
 import org.apache.http.client.config.CookieSpecs;
-import org.apache.http.impl.cookie.BestMatchSpecFactory;
-import org.apache.http.impl.cookie.BrowserCompatSpecFactory;
-import org.apache.http.impl.cookie.RFC2965SpecFactory;
-import org.apache.http.impl.cookie.NetscapeDraftSpecFactory;
-import org.apache.http.impl.cookie.IgnoreSpecFactory;
 
 import org.apache.http.cookie.MalformedCookieException;
 import org.apache.http.conn.ConnectTimeoutException;
@@ -147,11 +148,7 @@ public class ThrottledFetcher
 
   private static final Registry<CookieSpecProvider> cookieSpecRegistry =
     RegistryBuilder.<CookieSpecProvider>create()
-      .register(CookieSpecs.BEST_MATCH, new BestMatchSpecFactory())
-      .register(CookieSpecs.STANDARD, new RFC2965SpecFactory())
-      .register(CookieSpecs.BROWSER_COMPATIBILITY, new LaxBrowserCompatSpecFactory())
-      .register(CookieSpecs.NETSCAPE, new NetscapeDraftSpecFactory())
-      .register(CookieSpecs.IGNORE_COOKIES, new IgnoreSpecFactory())
+      .register(CookieSpecs.STANDARD, new LaxBrowserCompatSpecProvider())
       .build();
 
   /** Constructor.  Private since we never instantiate.
@@ -183,6 +180,7 @@ public class ThrottledFetcher
     IThrottleSpec throttleDescription, String[] binNames,
     int connectionLimit,
     String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword,
+    int socketTimeoutMilliseconds, int connectionTimeoutMilliseconds,
     IAbortActivity activities)
     throws ManifoldCFException, ServiceInterruption
   {
@@ -208,7 +206,8 @@ public class ThrottledFetcher
 
     // Construct a connection pool key
     ConnectionPoolKey poolKey = new ConnectionPoolKey(protocol,server,port,authentication,
-      trustStoreString,proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword);
+      trustStoreString,proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword,
+      socketTimeoutMilliseconds,connectionTimeoutMilliseconds);
     
     ConnectionPool p;
     synchronized (connectionPools)
@@ -220,7 +219,8 @@ public class ThrottledFetcher
         IConnectionThrottler connectionThrottler =
           throttleGroups.obtainConnectionThrottler(webThrottleGroupType,throttleGroupName,binNames);
         p = new ConnectionPool(connectionThrottler,protocol,server,port,authentication,baseFactory,
-          proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword);
+          proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword,
+          socketTimeoutMilliseconds,connectionTimeoutMilliseconds);
         connectionPools.put(poolKey,p);
       }
     }
@@ -294,6 +294,10 @@ public class ThrottledFetcher
     protected final String proxyAuthPassword;
     /** Https protocol */
     protected final javax.net.ssl.SSLSocketFactory httpsSocketFactory;
+    /** Socket timeout milliseconds */
+    protected final int socketTimeoutMilliseconds;
+    /** Connection timeout milliseconds */
+    protected final int connectionTimeoutMilliseconds;
 
     /** The thread that is actually doing the work */
     protected ExecuteMethodThread methodThread = null;
@@ -308,7 +312,8 @@ public class ThrottledFetcher
     public ThrottledConnection(ConnectionPool myPool, IFetchThrottler fetchThrottler,
       String protocol, String server, int port, PageCredentials authentication,
       javax.net.ssl.SSLSocketFactory httpsSocketFactory,
-      String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)
+      String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword,
+      int socketTimeoutMilliseconds, int connectionTimeoutMilliseconds)
     {
       this.myPool = myPool;
       this.fetchThrottler = fetchThrottler;
@@ -322,6 +327,8 @@ public class ThrottledFetcher
       this.port = port;
       this.authentication = authentication;
       this.httpsSocketFactory = httpsSocketFactory;
+      this.socketTimeoutMilliseconds = socketTimeoutMilliseconds;
+      this.connectionTimeoutMilliseconds = connectionTimeoutMilliseconds;
     }
 
     /** Set the abort checker.  This must be done before the connection is actually used.
@@ -402,21 +409,20 @@ public class ThrottledFetcher
     * @param urlPath is the path part of the url, e.g. "/robots.txt"
     * @param userAgent is the value of the userAgent header to use.
     * @param from is the value of the from header to use.
-    * @param connectionTimeoutMilliseconds is the maximum number of milliseconds to wait on socket connect.
     * @param redirectOK should be set to true if you want redirects to be automatically followed.
     * @param host is the value to use as the "Host" header, or null to use the default.
     * @param formData describes additional form arguments and how to fetch the page.
     * @param loginCookies describes the cookies that should be in effect for this page fetch.
     */
     @Override
-    public void executeFetch(String urlPath, String userAgent, String from, int connectionTimeoutMilliseconds,
-      int socketTimeoutMilliseconds, boolean redirectOK, String host, FormData formData,
+    public void executeFetch(String urlPath, String userAgent, String from,
+      boolean redirectOK, String host, FormData formData,
       LoginCookies loginCookies)
       throws ManifoldCFException, ServiceInterruption
     {
       // Set up scheme
       SSLConnectionSocketFactory myFactory = new SSLConnectionSocketFactory(new InterruptibleSocketFactory(httpsSocketFactory,connectionTimeoutMilliseconds),
-        SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+        NoopHostnameVerifier.INSTANCE);
 
       int hostPort;
       String displayedPort;
@@ -462,7 +468,14 @@ public class ThrottledFetcher
       
       if (connManager == null)
       {
-        connManager = new PoolingHttpClientConnectionManager();
+        PoolingHttpClientConnectionManager poolingConnManager = new PoolingHttpClientConnectionManager();
+        poolingConnManager.setDefaultMaxPerRoute(1);
+        poolingConnManager.setValidateAfterInactivity(60000);
+        poolingConnManager.setDefaultSocketConfig(SocketConfig.custom()
+          .setTcpNoDelay(true)
+          .setSoTimeout(socketTimeoutMilliseconds)
+          .build());
+        connManager = poolingConnManager;
       }
       
       long startTime = 0L;
@@ -486,11 +499,10 @@ public class ThrottledFetcher
       RequestConfig.Builder requestBuilder = RequestConfig.custom()
         .setCircularRedirectsAllowed(true)
         .setSocketTimeout(socketTimeoutMilliseconds)
-        .setStaleConnectionCheckEnabled(true)
         .setExpectContinueEnabled(true)
         .setConnectTimeout(connectionTimeoutMilliseconds)
         .setConnectionRequestTimeout(socketTimeoutMilliseconds)
-        .setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY)
+        .setCookieSpec(CookieSpecs.STANDARD)
         .setRedirectsEnabled(redirectOK);
 
       // If there's a proxy, set that too.
@@ -512,15 +524,8 @@ public class ThrottledFetcher
 
       httpClient = HttpClients.custom()
         .setConnectionManager(connManager)
-        .setMaxConnTotal(1)
-        .setMaxConnPerRoute(1)
         .disableAutomaticRetries()
-        .setDefaultCookieSpecRegistry(cookieSpecRegistry)
         .setDefaultRequestConfig(requestBuilder.build())
-        .setDefaultSocketConfig(SocketConfig.custom()
-          .setTcpNoDelay(true)
-          .setSoTimeout(socketTimeoutMilliseconds)
-          .build())
         .setDefaultCredentialsProvider(credentialsProvider)
         .setSSLSocketFactory(myFactory)
         .setRequestExecutor(new HttpRequestExecutor(socketTimeoutMilliseconds))
@@ -1312,35 +1317,14 @@ public class ThrottledFetcher
 
   /** Class to create a cookie spec.
   */
-  protected static class LaxBrowserCompatSpecFactory extends BrowserCompatSpecFactory
+  protected static class LaxBrowserCompatSpecProvider extends RFC6265CookieSpecProvider
   {
+    @Override
     public CookieSpec create(HttpContext context)
     {
       return new LaxBrowserCompatSpec();
     }
   }
-  
-  /** Class to override browser compatibility to make it not check cookie paths.  See CONNECTORS-97.
-  */
-  protected static class LaxBrowserCompatSpec extends BrowserCompatSpec
-  {
-
-    public LaxBrowserCompatSpec()
-    {
-      super();
-      registerAttribHandler(ClientCookie.PATH_ATTR, new BasicPathHandler()
-        {
-          @Override
-          public void validate(Cookie cookie, CookieOrigin origin) throws MalformedCookieException
-          {
-            // No validation
-          }
-              
-        }
-      );
-    }
-    
-  }
 
   /** This thread does the actual socket communication with the server.
   * It's set up so that it can be abandoned at shutdown time.
@@ -1859,11 +1843,14 @@ public class ThrottledFetcher
     protected final String proxyAuthDomain;
     protected final String proxyAuthUsername;
     protected final String proxyAuthPassword;
+    protected final int socketTimeoutMilliseconds;
+    protected final int connectionTimeoutMilliseconds;
     
     public ConnectionPoolKey(String protocol,
       String server, int port, PageCredentials authentication,
       String trustStoreString, String proxyHost, int proxyPort,
-      String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)
+      String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword,
+      int socketTimeoutMilliseconds, int connectionTimeoutMilliseconds)
     {
       this.protocol = protocol;
       this.server = server;
@@ -1875,6 +1862,8 @@ public class ThrottledFetcher
       this.proxyAuthDomain = proxyAuthDomain;
       this.proxyAuthUsername = proxyAuthUsername;
       this.proxyAuthPassword = proxyAuthPassword;
+      this.socketTimeoutMilliseconds = socketTimeoutMilliseconds;
+      this.connectionTimeoutMilliseconds = connectionTimeoutMilliseconds;
     }
     
     public int hashCode()
@@ -1888,7 +1877,9 @@ public class ThrottledFetcher
         (proxyPort * 29) +
         ((proxyAuthDomain==null)?0:proxyAuthDomain.hashCode()) +
         ((proxyAuthUsername==null)?0:proxyAuthUsername.hashCode()) +
-        ((proxyAuthPassword==null)?0:proxyAuthPassword.hashCode());
+        ((proxyAuthPassword==null)?0:proxyAuthPassword.hashCode()) +
+        new Integer(socketTimeoutMilliseconds).hashCode() +
+        new Integer(connectionTimeoutMilliseconds).hashCode();
     }
     
     public boolean equals(Object o)
@@ -1961,7 +1952,8 @@ public class ThrottledFetcher
         if (!proxyAuthPassword.equals(other.proxyAuthPassword))
           return false;
       }
-      return true;
+      return socketTimeoutMilliseconds == other.socketTimeoutMilliseconds &&
+        connectionTimeoutMilliseconds == other.connectionTimeoutMilliseconds;
     }
   }
   
@@ -1984,6 +1976,8 @@ public class ThrottledFetcher
     protected final String proxyAuthDomain;
     protected final String proxyAuthUsername;
     protected final String proxyAuthPassword;
+    protected final int socketTimeoutMilliseconds;
+    protected final int connectionTimeoutMilliseconds;
 
     /** The actual pool of connections */
     protected final List<IThrottledConnection> connections = new ArrayList<IThrottledConnection>();
@@ -1993,7 +1987,8 @@ public class ThrottledFetcher
       String server, int port, PageCredentials authentication,
       javax.net.ssl.SSLSocketFactory baseFactory,
       String proxyHost, int proxyPort,
-      String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)
+      String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword,
+      int socketTimeoutMilliseconds, int connectionTimeoutMilliseconds)
     {
       this.connectionThrottler = connectionThrottler;
       
@@ -2007,6 +2002,8 @@ public class ThrottledFetcher
       this.proxyAuthDomain = proxyAuthDomain;
       this.proxyAuthUsername = proxyAuthUsername;
       this.proxyAuthPassword = proxyAuthPassword;
+      this.socketTimeoutMilliseconds = socketTimeoutMilliseconds;
+      this.connectionTimeoutMilliseconds = connectionTimeoutMilliseconds;
     }
     
     public IThrottledConnection grab(IAbortActivity activities)
@@ -2031,7 +2028,8 @@ public class ThrottledFetcher
           connection = new ThrottledConnection(this,connectionThrottler.getNewConnectionFetchThrottler(),
             protocol,server,port,authentication,baseFactory,
             proxyHost,proxyPort,
-            proxyAuthDomain,proxyAuthUsername,proxyAuthPassword);
+            proxyAuthDomain,proxyAuthUsername,proxyAuthPassword,
+            socketTimeoutMilliseconds,connectionTimeoutMilliseconds);
         }
         else
           throw new IllegalStateException("Unexpected return value from waitConnectionAvailable(): "+result);

Modified: manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java?rev=1672616&r1=1672615&r2=1672616&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java (original)
+++ manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java Fri Apr 10 11:47:49 2015
@@ -856,6 +856,7 @@ public class WebcrawlerConnector extends
             protocol,ipAddress,port,
             credential,trustStore,throttleDescription,binNames,connectionLimit,
             proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword,
+            socketTimeoutMilliseconds,connectionTimeoutMilliseconds,
             activities);
           try
           {
@@ -863,8 +864,8 @@ public class WebcrawlerConnector extends
             try
             {
               // Execute the fetch!
-              connection.executeFetch(url.getFile(),userAgent,from,connectionTimeoutMilliseconds,
-                socketTimeoutMilliseconds,false,hostName,formData,lc);
+              connection.executeFetch(url.getFile(),userAgent,from,
+                false,hostName,formData,lc);
               int response = connection.getResponseCode();
 
               if (response == 200 || response == 302 || response == 301)
@@ -5345,13 +5346,14 @@ public class WebcrawlerConnector extends
           protocol,hostIPAddress,port,credential,
           trustStore,throttleDescription,binNames,connectionLimit,
           proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword,
+          socketTimeoutMilliseconds,connectionTimeoutMilliseconds,
           versionActivities);
         try
         {
           connection.beginFetch(FETCH_ROBOTS);
           try
           {
-            connection.executeFetch("/robots.txt",userAgent,from,connectionTimeoutMilliseconds,socketTimeoutMilliseconds,true,hostName,null,null);
+            connection.executeFetch("/robots.txt",userAgent,from,true,hostName,null,null);
             long expirationTime = currentTime+1000*60*60*24;
             int code = connection.getResponseCode();
             if (code == 200)

Modified: manifoldcf/trunk/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java?rev=1672616&r1=1672615&r2=1672616&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java (original)
+++ manifoldcf/trunk/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java Fri Apr 10 11:47:49 2015
@@ -65,6 +65,7 @@ import org.apache.http.message.BasicHead
 import org.apache.http.client.HttpRequestRetryHandler;
 import org.apache.http.protocol.HttpContext;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.entity.ContentType;
 
 import org.apache.http.conn.ConnectTimeoutException;
@@ -227,10 +228,17 @@ public class WikiConnector extends org.a
 
       javax.net.ssl.SSLSocketFactory httpsSocketFactory = KeystoreManagerFactory.getTrustingSecureSocketFactory();
       SSLConnectionSocketFactory myFactory = new SSLConnectionSocketFactory(new InterruptibleSocketFactory(httpsSocketFactory,connectionTimeout),
-        SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+        NoopHostnameVerifier.INSTANCE);
 
       // Set up connection manager
-      connectionManager = new PoolingHttpClientConnectionManager();
+      PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+      poolingConnectionManager.setDefaultMaxPerRoute(1);
+      poolingConnectionManager.setValidateAfterInactivity(60000);
+      poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+        .setTcpNoDelay(true)
+        .setSoTimeout(socketTimeout)
+        .build());
+      connectionManager = poolingConnectionManager;
 
       CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
 
@@ -246,7 +254,6 @@ public class WikiConnector extends org.a
       RequestConfig.Builder requestBuilder = RequestConfig.custom()
           .setCircularRedirectsAllowed(true)
           .setSocketTimeout(socketTimeout)
-          .setStaleConnectionCheckEnabled(true)
           .setExpectContinueEnabled(true)
           .setConnectTimeout(connectionTimeout)
           .setConnectionRequestTimeout(socketTimeout);
@@ -289,13 +296,8 @@ public class WikiConnector extends org.a
 
       httpClient = HttpClients.custom()
         .setConnectionManager(connectionManager)
-        .setMaxConnTotal(1)
         .disableAutomaticRetries()
         .setDefaultRequestConfig(requestBuilder.build())
-        .setDefaultSocketConfig(SocketConfig.custom()
-          .setTcpNoDelay(true)
-          .setSoTimeout(socketTimeout)
-          .build())
         .setDefaultCredentialsProvider(credentialsProvider)
         .setSSLSocketFactory(myFactory)
         .setRequestExecutor(new HttpRequestExecutor(socketTimeout))

Added: manifoldcf/trunk/framework/connector-common/src/main/java/org/apache/http/impl/cookie/LaxBrowserCompatSpec.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/connector-common/src/main/java/org/apache/http/impl/cookie/LaxBrowserCompatSpec.java?rev=1672616&view=auto
==============================================================================
--- manifoldcf/trunk/framework/connector-common/src/main/java/org/apache/http/impl/cookie/LaxBrowserCompatSpec.java (added)
+++ manifoldcf/trunk/framework/connector-common/src/main/java/org/apache/http/impl/cookie/LaxBrowserCompatSpec.java Fri Apr 10 11:47:49 2015
@@ -0,0 +1,51 @@
+/* $Id$ */
+
+/**`
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.http.impl.cookie;
+
+import org.apache.http.cookie.CookieOrigin;
+import org.apache.http.cookie.ClientCookie;
+import org.apache.http.cookie.CookieSpecProvider;
+import org.apache.http.cookie.Cookie;
+import org.apache.http.cookie.MalformedCookieException;
+
+/** Class to override browser compatibility to make it not check cookie paths.  See CONNECTORS-97.
+* The class must be in the package described because it requires a protected constructor from the
+* class it extends.
+*/
+public class LaxBrowserCompatSpec extends RFC6265LaxSpec
+{
+
+  public LaxBrowserCompatSpec()
+  {
+    super(new BasicPathHandler()
+    {
+      @Override
+      public void validate(Cookie cookie, CookieOrigin origin) throws MalformedCookieException
+      {
+        // No validation
+      }
+    },
+    new BasicDomainHandler(),
+    new LaxMaxAgeHandler(),
+    new BasicSecureHandler(),
+    new LaxExpiresHandler());
+  }
+    
+}

Propchange: manifoldcf/trunk/framework/connector-common/src/main/java/org/apache/http/impl/cookie/LaxBrowserCompatSpec.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: manifoldcf/trunk/framework/connector-common/src/main/java/org/apache/http/impl/cookie/LaxBrowserCompatSpec.java
------------------------------------------------------------------------------
    svn:keywords = Id

Modified: manifoldcf/trunk/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyShutdown.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyShutdown.java?rev=1672616&r1=1672615&r2=1672616&view=diff
==============================================================================
--- manifoldcf/trunk/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyShutdown.java (original)
+++ manifoldcf/trunk/framework/jetty-runner/src/main/java/org/apache/manifoldcf/jettyrunner/ManifoldCFJettyShutdown.java Fri Apr 10 11:47:49 2015
@@ -71,25 +71,26 @@ public class ManifoldCFJettyShutdown
       int socketTimeout = 900000;
       int connectionTimeout = 300000;
 
-      HttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
+      PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+      poolingConnectionManager.setDefaultMaxPerRoute(1);
+      poolingConnectionManager.setValidateAfterInactivity(60000);
+      poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+        .setTcpNoDelay(true)
+        .setSoTimeout(socketTimeout)
+        .build());
+      HttpClientConnectionManager connectionManager = poolingConnectionManager;
         
       RequestConfig.Builder requestBuilder = RequestConfig.custom()
         .setCircularRedirectsAllowed(true)
         .setSocketTimeout(socketTimeout)
-        .setStaleConnectionCheckEnabled(false)
         .setExpectContinueEnabled(true)
         .setConnectTimeout(connectionTimeout)
         .setConnectionRequestTimeout(socketTimeout);
 
       HttpClient httpClient = HttpClients.custom()
         .setConnectionManager(connectionManager)
-        .setMaxConnTotal(1)
         .disableAutomaticRetries()
         .setDefaultRequestConfig(requestBuilder.build())
-        .setDefaultSocketConfig(SocketConfig.custom()
-          .setTcpNoDelay(true)
-          .setSoTimeout(socketTimeout)
-          .build())
         .setRequestExecutor(new HttpRequestExecutor(socketTimeout))
         .setRedirectStrategy(new DefaultRedirectStrategy())
         .build();

Modified: manifoldcf/trunk/framework/script-engine/src/main/java/org/apache/manifoldcf/scriptengine/ScriptParser.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/script-engine/src/main/java/org/apache/manifoldcf/scriptengine/ScriptParser.java?rev=1672616&r1=1672615&r2=1672616&view=diff
==============================================================================
--- manifoldcf/trunk/framework/script-engine/src/main/java/org/apache/manifoldcf/scriptengine/ScriptParser.java (original)
+++ manifoldcf/trunk/framework/script-engine/src/main/java/org/apache/manifoldcf/scriptengine/ScriptParser.java Fri Apr 10 11:47:49 2015
@@ -1221,14 +1221,20 @@ public class ScriptParser
         int socketTimeout = 900000;
         int connectionTimeout = 300000;
 
-        connectionManager = new PoolingHttpClientConnectionManager();
+        PoolingHttpClientConnectionManager poolingConnectionManager = new PoolingHttpClientConnectionManager();
+        poolingConnectionManager.setDefaultMaxPerRoute(1);
+        poolingConnectionManager.setValidateAfterInactivity(60000);
+        poolingConnectionManager.setDefaultSocketConfig(SocketConfig.custom()
+          .setTcpNoDelay(true)
+          .setSoTimeout(socketTimeout)
+          .build());
+        connectionManager = poolingConnectionManager;
         
         CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
 
         RequestConfig.Builder requestBuilder = RequestConfig.custom()
           .setCircularRedirectsAllowed(true)
           .setSocketTimeout(socketTimeout)
-          .setStaleConnectionCheckEnabled(false)
           .setExpectContinueEnabled(true)
           .setConnectTimeout(connectionTimeout)
           .setConnectionRequestTimeout(socketTimeout);
@@ -1238,10 +1244,6 @@ public class ScriptParser
           .setMaxConnTotal(1)
           .disableAutomaticRetries()
           .setDefaultRequestConfig(requestBuilder.build())
-          .setDefaultSocketConfig(SocketConfig.custom()
-            .setTcpNoDelay(true)
-            .setSoTimeout(socketTimeout)
-            .build())
           .setDefaultCredentialsProvider(credentialsProvider)
           //.setSSLSocketFactory(myFactory)
           .setRequestExecutor(new HttpRequestExecutor(socketTimeout))



Mime
View raw message