manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1411890 - in /manifoldcf/branches/CONNECTORS-120/connectors: rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/ webcrawler/conne...
Date Tue, 20 Nov 2012 21:27:04 GMT
Author: kwright
Date: Tue Nov 20 21:27:02 2012
New Revision: 1411890

URL: http://svn.apache.org/viewvc?rev=1411890&view=rev
Log:
More work porting connectors to httpcomponents

Modified:
    manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
    manifoldcf/branches/CONNECTORS-120/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
    manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CookieManager.java
    manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CookieSet.java
    manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CredentialsDescription.java
    manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/LoginCookies.java
    manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/PageCredentials.java
    manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
    manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java

Modified: manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java?rev=1411890&r1=1411889&r2=1411890&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java (original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java Tue Nov 20 21:27:02 2012
@@ -50,6 +50,7 @@ import org.apache.http.HttpHost;
 import org.apache.http.Header;
 import org.apache.http.conn.params.ConnRoutePNames;
 import org.apache.http.message.BasicHeader;
+import org.apache.http.client.params.ClientPNames;
 
 import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.client.CircularRedirectException;
@@ -272,6 +273,7 @@ public class ThrottledFetcher
       params.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,false);
       params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,connectionTimeoutMilliseconds);
       params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,connectionTimeoutMilliseconds);
+      params.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
       DefaultHttpClient localHttpClient = new DefaultHttpClient(connectionManager,params);
       localHttpClient.setRedirectStrategy(new DefaultRedirectStrategy());
       

Modified: manifoldcf/branches/CONNECTORS-120/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java?rev=1411890&r1=1411889&r2=1411890&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java (original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/SharePointRepository.java Tue Nov 20 21:27:02 2012
@@ -54,7 +54,7 @@ import org.apache.http.util.EntityUtils;
 import org.apache.http.params.BasicHttpParams;
 import org.apache.http.params.HttpParams;
 import org.apache.http.params.CoreConnectionPNames;
-
+import org.apache.http.client.params.ClientPNames;
 
 /** This is the "repository connector" for Microsoft SharePoint.
 * Document identifiers for this connector come in three forms:
@@ -219,6 +219,7 @@ public class SharePointRepository extend
       params.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY,true);
       params.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,false);
       params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,60000);
+      params.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
       DefaultHttpClient localHttpClient = new DefaultHttpClient(connectionManager,params);
       localHttpClient.setRedirectStrategy(new DefaultRedirectStrategy());
       if (strippedUserName != null)

Modified: manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CookieManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CookieManager.java?rev=1411890&r1=1411889&r2=1411890&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CookieManager.java (original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CookieManager.java Tue Nov 20 21:27:02 2012
@@ -27,8 +27,9 @@ import org.apache.manifoldcf.crawler.int
 import org.apache.manifoldcf.crawler.system.ManifoldCF;
 import org.apache.manifoldcf.crawler.system.Logging;
 
-import org.apache.commons.httpclient.Cookie;
-import org.apache.commons.httpclient.cookie.Cookie2;
+import org.apache.http.cookie.Cookie;
+import org.apache.http.cookie.ClientCookie;
+import org.apache.http.impl.cookie.BasicClientCookie2;
 
 /** This class manages the database table into which we write cookies.  The data resides in the database,
 * as well as in cache (up to a certain point).  The result is that there is a memory limited, database-backed repository
@@ -217,18 +218,13 @@ public class CookieManager extends org.a
         while (i < cookies.getCookieCount())
         {
           Cookie c = cookies.getCookie(i);
-          Cookie2 c2;
-          if (c instanceof Cookie2)
-            c2 = (Cookie2)c;
-          else
-            c2 = null;
           HashMap map = new HashMap();
           map.put(keyField,sessionKey);
           map.put(ordinalField,new Long(i));
           String domain = c.getDomain();
           if (domain != null && domain.length() > 0)
             map.put(domainField,domain);
-          map.put(domainSpecifiedField,booleanToString(c.isDomainAttributeSpecified()));
+          map.put(domainSpecifiedField,booleanToString(domain != null && domain.length() > 0));
           String name = c.getName();
           if (name != null && name.length() > 0)
             map.put(nameField,name);
@@ -238,45 +234,26 @@ public class CookieManager extends org.a
           String path = c.getPath();
           if (path != null && path.length() > 0)
             map.put(pathField,path);
-          map.put(pathSpecifiedField,booleanToString(c.isPathAttributeSpecified()));
+          map.put(pathSpecifiedField,booleanToString(path != null && path.length() > 0));
           map.put(versionField,new Long(c.getVersion()));
-          if (c2 != null)
-            map.put(versionSpecifiedField,booleanToString(c2.isVersionAttributeSpecified()));
-          else
-            // Make something up.  It may not be correct, but there's really no choice.
-            map.put(versionSpecifiedField,booleanToString(true));
+          // Make something up.  It may not be correct, but there's really no choice.
+          map.put(versionSpecifiedField,booleanToString(true));
           String comment = c.getComment();
           if (comment != null && comment.length() > 0)
             map.put(commentField,comment);
-          map.put(secureField,booleanToString(c.getSecure()));
+          map.put(secureField,booleanToString(c.isSecure()));
           Date expirationDate = c.getExpiryDate();
           if (expirationDate != null)
             map.put(expirationDateField,new Long(expirationDate.getTime()));
-          if (c2 != null)
-            map.put(discardField,booleanToString(!c2.isPersistent()));
-          else
-            // Once again, make something up.
-            map.put(discardField,booleanToString(true));
-          if (c2 != null)
-          {
-            String commentURL = c2.getCommentURL();
-            if (commentURL != null && commentURL.length() > 0)
-              map.put(commentURLField,commentURL);
-          }
-          if (c2 != null)
-          {
-            int[] ports = c2.getPorts();
-            if (ports != null && ports.length > 0)
-              map.put(portField,portsToString(ports));
-          }
-          if (c2 != null)
-            map.put(portBlankField,booleanToString(c2.isPortAttributeBlank()));
-          else
-            map.put(portBlankField,booleanToString(true));
-          if (c2 != null)
-            map.put(portSpecifiedField,booleanToString(c2.isPortAttributeSpecified()));
-          else
-            map.put(portSpecifiedField,booleanToString(false));
+          map.put(discardField,booleanToString(!c.isPersistent()));
+          String commentURL = c.getCommentURL();
+          if (commentURL != null && commentURL.length() > 0)
+            map.put(commentURLField,commentURL);
+          int[] ports = c.getPorts();
+          if (ports != null && ports.length > 0)
+            map.put(portField,portsToString(ports));
+          map.put(portBlankField,booleanToString(ports == null || ports.length == 0));
+          map.put(portSpecifiedField,booleanToString(ports != null && ports.length > 0));
           performInsert(map,null);
           i++;
         }
@@ -330,25 +307,21 @@ public class CookieManager extends org.a
     while (i < result.getRowCount())
     {
       IResultRow row = result.getRow(i++);
-      Cookie2 c = new Cookie2();
+      String name = (String)row.getValue(nameField);
+      String value = (String)row.getValue(valueField);
+      BasicClientCookie2 c = new BasicClientCookie2(name,value);
       String domain = (String)row.getValue(domainField);
       if (domain != null && domain.length() > 0)
         c.setDomain(domain);
-      c.setDomainAttributeSpecified(stringToBoolean((String)row.getValue(domainSpecifiedField)));
-      String name = (String)row.getValue(nameField);
-      if (name != null && name.length() > 0)
-        c.setName(name);
-      String value = (String)row.getValue(valueField);
-      if (value != null && value.length() > 0)
-        c.setValue(value);
+      //c.setDomainAttributeSpecified(stringToBoolean((String)row.getValue(domainSpecifiedField)));
       String path = (String)row.getValue(pathField);
       if (path != null && path.length() > 0)
         c.setPath(path);
-      c.setPathAttributeSpecified(stringToBoolean((String)row.getValue(pathSpecifiedField)));
+      //c.setPathAttributeSpecified(stringToBoolean((String)row.getValue(pathSpecifiedField)));
       Long version = (Long)row.getValue(versionField);
       if (version != null)
         c.setVersion((int)version.longValue());
-      c.setVersionAttributeSpecified(stringToBoolean((String)row.getValue(versionSpecifiedField)));
+      //c.setVersionAttributeSpecified(stringToBoolean((String)row.getValue(versionSpecifiedField)));
       String comment = (String)row.getValue(commentField);
       if (comment != null)
         c.setComment(comment);
@@ -364,8 +337,8 @@ public class CookieManager extends org.a
       // Ports are comma-separated
       if (ports != null && ports.length() > 0)
         c.setPorts(stringToPorts(ports));
-      c.setPortAttributeBlank(stringToBoolean((String)row.getValue(portBlankField)));
-      c.setPortAttributeSpecified(stringToBoolean((String)row.getValue(portSpecifiedField)));
+      //c.setPortAttributeBlank(stringToBoolean((String)row.getValue(portBlankField)));
+      //c.setPortAttributeSpecified(stringToBoolean((String)row.getValue(portSpecifiedField)));
 
       dcs.addCookie(c);
     }
@@ -438,7 +411,7 @@ public class CookieManager extends org.a
   */
   protected static class DynamicCookieSet implements LoginCookies
   {
-    protected ArrayList cookies = new ArrayList();
+    protected List<Cookie> cookies = new ArrayList<Cookie>();
 
     public DynamicCookieSet()
     {
@@ -456,7 +429,7 @@ public class CookieManager extends org.a
 
     public Cookie getCookie(int index)
     {
-      return (Cookie)cookies.get(index);
+      return cookies.get(index);
     }
   }
 

Modified: manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CookieSet.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CookieSet.java?rev=1411890&r1=1411889&r2=1411890&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CookieSet.java (original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CookieSet.java Tue Nov 20 21:27:02 2012
@@ -18,25 +18,27 @@
 */
 package org.apache.manifoldcf.crawler.connectors.webcrawler;
 
-import org.apache.commons.httpclient.*;
+import java.util.List;
+
+import org.apache.http.cookie.Cookie;
 
 /** This class represents a bunch of cookies */
 public class CookieSet implements LoginCookies
 {
-  protected Cookie[] cookieList;
+  protected List<Cookie> cookieList;
 
-  public CookieSet(Cookie[] cookieList)
+  public CookieSet(List<Cookie> cookieList)
   {
     this.cookieList = cookieList;
   }
 
   public int getCookieCount()
   {
-    return cookieList.length;
+    return cookieList.size();
   }
 
   public Cookie getCookie(int index)
   {
-    return cookieList[index];
+    return cookieList.get(index);
   }
 }

Modified: manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CredentialsDescription.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CredentialsDescription.java?rev=1411890&r1=1411889&r2=1411890&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CredentialsDescription.java (original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CredentialsDescription.java Tue Nov 20 21:27:02 2012
@@ -23,9 +23,9 @@ import org.apache.manifoldcf.crawler.sys
 import java.util.*;
 import java.util.regex.*;
 
-import org.apache.commons.httpclient.*;
-import org.apache.commons.httpclient.methods.*;
-import org.apache.commons.httpclient.params.*;
+import org.apache.http.auth.Credentials;
+import org.apache.http.auth.NTCredentials;
+import org.apache.http.auth.UsernamePasswordCredentials;
 
 /** This class describes credential information pulled from a configuration.
 * The data contained is organized by regular expression performed on a url.  What we store

Modified: manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/LoginCookies.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/LoginCookies.java?rev=1411890&r1=1411889&r2=1411890&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/LoginCookies.java (original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/LoginCookies.java Tue Nov 20 21:27:02 2012
@@ -18,7 +18,7 @@
 */
 package org.apache.manifoldcf.crawler.connectors.webcrawler;
 
-import org.apache.commons.httpclient.*;
+import org.apache.http.cookie.Cookie;
 
 /** This interface describes cookies obtained during sequential authentication.
 */

Modified: manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/PageCredentials.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/PageCredentials.java?rev=1411890&r1=1411889&r2=1411890&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/PageCredentials.java (original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/PageCredentials.java Tue Nov 20 21:27:02 2012
@@ -20,9 +20,7 @@ package org.apache.manifoldcf.crawler.co
 
 import org.apache.manifoldcf.core.interfaces.*;
 
-import org.apache.commons.httpclient.*;
-import org.apache.commons.httpclient.methods.*;
-import org.apache.commons.httpclient.params.*;
+import org.apache.http.auth.Credentials;
 
 /** This interface describes immutable classes which represents authentication information for page-based authentication.
 */

Modified: manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java?rev=1411890&r1=1411889&r2=1411890&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java (original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java Tue Nov 20 21:27:02 2012
@@ -1,6 +1,6 @@
 /* $Id: ThrottledFetcher.java 989847 2010-08-26 17:52:30Z kwright $ */
 
-/**
+/**`
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
@@ -19,6 +19,7 @@
 package org.apache.manifoldcf.crawler.connectors.webcrawler;
 
 import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.core.common.XThreadInputStream;
 import org.apache.manifoldcf.agents.interfaces.*;
 import org.apache.manifoldcf.crawler.interfaces.*;
 import org.apache.manifoldcf.crawler.system.Logging;
@@ -26,13 +27,49 @@ import org.apache.manifoldcf.crawler.sys
 import java.util.*;
 import java.io.*;
 import java.net.*;
+import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.httpclient.*;
-import org.apache.commons.httpclient.methods.*;
-import org.apache.commons.httpclient.params.*;
-import org.apache.commons.httpclient.protocol.*;
-import org.apache.commons.httpclient.cookie.CookiePolicy;
-import org.apache.commons.httpclient.auth.AuthScope;
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.NameValuePair;
+import org.apache.http.impl.conn.PoolingClientConnectionManager;
+import org.apache.http.conn.scheme.Scheme;
+import org.apache.http.conn.ssl.SSLSocketFactory;
+import org.apache.http.conn.ssl.BrowserCompatHostnameVerifier;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.NTCredentials;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.impl.client.AbstractHttpClient;
+import org.apache.http.impl.client.DefaultRedirectStrategy;
+import org.apache.http.util.EntityUtils;
+import org.apache.http.params.BasicHttpParams;
+import org.apache.http.params.HttpParams;
+import org.apache.http.params.CoreConnectionPNames;
+import org.apache.http.HttpStatus;
+import org.apache.http.HttpHost;
+import org.apache.http.Header;
+import org.apache.http.conn.params.ConnRoutePNames;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.client.params.ClientPNames;
+import org.apache.http.client.params.HttpClientParams;
+import org.apache.http.client.params.CookiePolicy;
+import org.apache.http.cookie.params.CookieSpecPNames;
+import org.apache.http.impl.cookie.BasicClientCookie;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.protocol.HTTP;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+
+import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.http.client.RedirectException;
+import org.apache.http.client.CircularRedirectException;
+import org.apache.http.NoHttpResponseException;
+import org.apache.http.HttpException;
 
 /** This class uses httpclient to fetch stuff from webservers.  However, it additionally controls the fetch
 * rate in two ways: first, controlling the overall bandwidth used per server, and second, limiting the number
@@ -114,23 +151,21 @@ public class ThrottledFetcher
     String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)
     throws ManifoldCFException
   {
-    // First, create a protocol factory object, if we can
-    ProtocolFactory myFactory = new ProtocolFactory();
+    // Create the https scheme for this connection
+    SSLSocketFactory myFactory;
     String trustStoreString;
-    ProtocolSocketFactory secureSocketFactory;
     if (trustStore != null)
     {
-      secureSocketFactory = new WebSecureSocketFactory(trustStore.getSecureSocketFactory());
+      myFactory = new SSLSocketFactory(trustStore.getSecureSocketFactory(), new BrowserCompatHostnameVerifier());
       trustStoreString = trustStore.getString();
     }
     else
     {
+      myFactory = new SSLSocketFactory(KeystoreManagerFactory.getTrustingSecureSocketFactory(), new BrowserCompatHostnameVerifier());
       trustStoreString = null;
-      secureSocketFactory = new WebSecureSocketFactory(KeystoreManagerFactory.getTrustingSecureSocketFactory());
     }
-
-    Protocol myHttpsProtocol = new Protocol("https", (ProtocolSocketFactory)secureSocketFactory, 443);
-    myFactory.registerProtocol("https",myHttpsProtocol);
+    
+    Scheme myHttpsProtocol = new Scheme("https", 443, myFactory);
 
     ConnectionBin[] bins = new ConnectionBin[binNames.length];
 
@@ -370,7 +405,7 @@ public class ThrottledFetcher
 
           // If we have a connection located, activate it.
           if (connectionToReuse == null)
-            connectionToReuse = new ThrottledConnection(protocol,server,port,authentication,myFactory,trustStoreString,bins,
+            connectionToReuse = new ThrottledConnection(protocol,server,port,authentication,myHttpsProtocol,trustStoreString,bins,
               proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword);
           connectionToReuse.setup(throttleDescription);
           return connectionToReuse;
@@ -853,126 +888,6 @@ public class ThrottledFetcher
 
   }
 
-  // Where to record result info/file data
-  protected final static String resultLogFile = "/common/web/resultlog";
-  // Where to record the actual data
-  protected final static String dataFileFolder = "/common/web/data/";
-
-  // This is the one instance of the output class
-  protected static DataRecorder dataRecorder = new DataRecorder();
-
-  /** This class takes care of recording data and results for posterity */
-  protected static class DataRecorder
-  {
-    protected int documentNumber = 0;
-
-    public DataRecorder()
-    {
-    }
-
-    public DataSession getSession(String url)
-      throws ManifoldCFException
-    {
-      return new DataSession(this,url);
-    }
-
-    /** Atomically write resultlog record, returning data file name to use */
-    public synchronized String writeResponseRecord(String url, int responseCode, ArrayList headerNames, ArrayList headerValues)
-      throws ManifoldCFException
-    {
-      // Open log file
-      try
-      {
-        OutputStream os = new FileOutputStream(resultLogFile,true);
-        try
-        {
-          OutputStreamWriter writer = new OutputStreamWriter(os,"utf-8");
-          try
-          {
-            String documentName = Integer.toString(documentNumber++);
-            writer.write("URI: "+url+"\n");
-            writer.write("File: "+documentName+"\n");
-            writer.write("Code: "+Integer.toString(responseCode)+"\n");
-            int i = 0;
-            while (i < headerNames.size())
-            {
-              writer.write("Header: "+(String)headerNames.get(i)+":"+(String)headerValues.get(i)+"\n");
-              i++;
-            }
-            return documentName;
-          }
-          finally
-          {
-            writer.close();
-          }
-        }
-        finally
-        {
-          os.close();
-        }
-      }
-      catch (IOException e)
-      {
-        throw new ManifoldCFException("Error recording file info: "+e.getMessage(),e);
-      }
-
-    }
-
-
-  }
-
-  /** Helper class for the above */
-  protected static class DataSession
-  {
-    protected DataRecorder dr;
-    protected String url;
-    protected int responseCode = 0;
-    protected ArrayList headerNames = new ArrayList();
-    protected ArrayList headerValues = new ArrayList();
-    protected String documentName = null;
-
-    public DataSession(DataRecorder dr, String url)
-    {
-      this.dr = dr;
-      this.url = url;
-    }
-
-    public void setResponseCode(int responseCode)
-    {
-      this.responseCode = responseCode;
-    }
-
-    public void addHeader(String headerName, String headerValue)
-    {
-      headerNames.add(headerName);
-      headerValues.add(headerValue);
-    }
-
-    public void endHeader()
-      throws ManifoldCFException
-    {
-      documentName = dr.writeResponseRecord(url,responseCode,headerNames,headerValues);
-    }
-
-    public void write(byte[] theBytes, int off, int length)
-      throws IOException
-    {
-      if (documentName == null)
-        throw new IOException("Must end header before reading data!");
-      OutputStream os = new FileOutputStream(dataFileFolder+documentName,true);
-      try
-      {
-        os.write(theBytes,off,length);
-      }
-      finally
-      {
-        os.close();
-      }
-    }
-
-  }
-
-
   /** Throttled connections.  Each instance of a connection describes the bins to which it belongs,
   * along with the actual open connection itself, and the last time the connection was used. */
   protected static class ThrottledConnection implements IThrottledConnection
@@ -1004,9 +919,11 @@ public class ThrottledFetcher
     protected String trustStoreString;
 
     /** The http connection manager.  The pool is of size 1.  */
-    protected MultiThreadedHttpConnectionManager connManager = null;
+    protected PoolingClientConnectionManager connManager = null;
+    /** The http client object. */
+    protected AbstractHttpClient httpClient = null;
     /** The method object */
-    protected HttpMethodBase fetchMethod = null;
+    protected HttpRequestBase fetchMethod = null;
     /** The error trace, if any */
     protected Throwable throwable = null;
     /** The current URL being fetched */
@@ -1031,19 +948,19 @@ public class ThrottledFetcher
     protected final String proxyAuthUsername;
     /** Proxy auth password */
     protected final String proxyAuthPassword;
-    
-    /** Protocol socket factory */
-    protected ProtocolSocketFactory secureSocketFactory = null;
-    protected ProtocolFactory myFactory = null;
+    /** Https protocol */
+    protected final Scheme myHttpsProtocol;
 
-
-    /** Hack added to record all access data from current crawler */
-    protected DataSession dataSession = null;
+    /** The thread that is actually doing the work */
+    protected ExecuteMethodThread methodThread = null;
+    /** Set if thread has been started */
+    protected boolean threadStarted = false;
+    
 
     /** Constructor.  Create a connection with a specific server and port, and
     * register it as active against all bins. */
     public ThrottledConnection(String protocol, String server, int port, PageCredentials authentication,
-      ProtocolFactory myFactory, String trustStoreString, ConnectionBin[] connectionBins,
+      Scheme myHttpsProtocol, String trustStoreString, ConnectionBin[] connectionBins,
       String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)
     {
       this.proxyHost = proxyHost;
@@ -1055,7 +972,7 @@ public class ThrottledFetcher
       this.server = server;
       this.port = port;
       this.authentication = authentication;
-      this.myFactory = myFactory;
+      this.myHttpsProtocol = myHttpsProtocol;
       this.trustStoreString = trustStoreString;
       this.connectionBinArray = connectionBins;
       this.throttleBinArray = new ThrottleBin[connectionBins.length];
@@ -1210,10 +1127,11 @@ public class ThrottledFetcher
 
       if (connManager != null)
       {
-        connManager.closeIdleConnections(idleTimeout);
-        connManager.deleteClosedConnections();
+        connManager.closeIdleConnections(idleTimeout, TimeUnit.MILLISECONDS);
+        connManager.closeExpiredConnections();
         // Need to determine if there's a valid connection in the connection manager still, or if it is empty.
-        return connManager.getConnectionsInPool() == 0;
+        //return connManager.getConnectionsInPool() == 0;
+        return true;
       }
       else
         return true;
@@ -1312,47 +1230,6 @@ public class ThrottledFetcher
       }
     }
 
-    protected static class ExecuteMethodThread extends Thread
-    {
-      protected HttpClient client;
-      protected HostConfiguration hostConfiguration;
-      protected HttpMethodBase executeMethod;
-      protected Throwable exception = null;
-      protected int rval = 0;
-
-      public ExecuteMethodThread(HttpClient client, HostConfiguration hostConfiguration, HttpMethodBase executeMethod)
-      {
-        super();
-        setDaemon(true);
-        this.client = client;
-        this.hostConfiguration = hostConfiguration;
-        this.executeMethod = executeMethod;
-      }
-
-      public void run()
-      {
-        try
-        {
-          // Call the execute method appropriately
-          rval = client.executeMethod(hostConfiguration,executeMethod,null);
-        }
-        catch (Throwable e)
-        {
-          this.exception = e;
-        }
-      }
-
-      public Throwable getException()
-      {
-        return exception;
-      }
-
-      public int getResponse()
-      {
-        return rval;
-      }
-    }
-
     /** Execute the fetch and get the return code.  This method uses the
     * standard logging mechanism to keep track of the fetch attempt.  It also
     * signals the following conditions: ServiceInterruption (if a dynamic
@@ -1401,199 +1278,206 @@ public class ThrottledFetcher
       else
         myUrl = fetchUrl;
 
-      if (recordEverything)
-        // Start a new data session
-        dataSession = dataRecorder.getSession(myUrl);
-
-      try
+      if (connManager == null)
       {
-        if (connManager == null)
-          connManager = new MultiThreadedHttpConnectionManager();
-        HttpConnectionManagerParams httpConParam = connManager.getParams();
-        httpConParam.setDefaultMaxConnectionsPerHost(1);
-        httpConParam.setMaxTotalConnections(1);
-        httpConParam.setConnectionTimeout(connectionTimeoutMilliseconds);
-        httpConParam.setSoTimeout(socketTimeoutMilliseconds);
-        connManager.setParams(httpConParam);
-
-        long startTime = 0L;
-        if (Logging.connectors.isDebugEnabled())
-        {
-          startTime = System.currentTimeMillis();
-          Logging.connectors.debug("WEB: Waiting for an HttpClient object");
-        }
-
-
-        HttpClient client = new HttpClient(connManager);
-        // Permit circular redirections, because that is how some sites set cookies
-        client.getParams().setParameter(org.apache.commons.httpclient.params.HttpClientParams.ALLOW_CIRCULAR_REDIRECTS,new Boolean(true));
-        // If there are redirects, this is essential to make sure the right socket factory gets used
-        client.getParams().setParameter(org.apache.commons.httpclient.params.HttpClientParams.PROTOCOL_FACTORY,myFactory);
-
-        HostConfiguration clientConf = new HostConfiguration();
-        // clientConf.setLocalAddress(currentAddr);
-
-        // Set up protocol to use
-        clientConf.setParams(new HostParams());
-        clientConf.setHost(server,port,myFactory.getProtocol(protocol));
+        PoolingClientConnectionManager localConnManager = new PoolingClientConnectionManager();
+        // Set up protocol registry
+        localConnManager.getSchemeRegistry().register(myHttpsProtocol);
+        localConnManager.setMaxTotal(1);
+        localConnManager.setDefaultMaxPerRoute(1);
+        connManager = localConnManager;
+      }
+
+      long startTime = 0L;
+      if (Logging.connectors.isDebugEnabled())
+      {
+        startTime = System.currentTimeMillis();
+        Logging.connectors.debug("WEB: Waiting for an HttpClient object");
+      }
+
+      // If we already have an httpclient object, great.  Otherwise we have to get one, and initialize it with
+      // those parameters that aren't expected to change.
+      if (httpClient == null)
+      {
+        BasicHttpParams params = new BasicHttpParams();
+        params.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY,true);
+        params.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,false);
+        params.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
+        // MEDIUM_SECURITY compatibility level not supported in HttpComponents.  Try BROWSER_NETSCAPE?
+        HttpClientParams.setCookiePolicy(params,CookiePolicy.BROWSER_COMPATIBILITY);
+        params.setBooleanParameter(CookieSpecPNames.SINGLE_COOKIE_HEADER,new Boolean(true));
+
+        DefaultHttpClient localHttpClient = new DefaultHttpClient(connManager,params);
+        localHttpClient.setRedirectStrategy(new DefaultRedirectStrategy());
+          
         // If there's a proxy, set that too.
         if (proxyHost != null && proxyHost.length() > 0)
         {
-          clientConf.setProxy(proxyHost,proxyPort);
+          // Configure proxy authentication
           if (proxyAuthUsername != null && proxyAuthUsername.length() > 0)
           {
-            // Set up NTLM credentials for this fetch too.
-            client.getState().setProxyCredentials(AuthScope.ANY,
-              new NTCredentials(proxyAuthUsername,(proxyAuthPassword==null)?"":proxyAuthPassword,currentHost,(proxyAuthDomain==null)?"":proxyAuthDomain));
+            localHttpClient.getCredentialsProvider().setCredentials(
+              new AuthScope(proxyHost, proxyPort),
+              new NTCredentials(proxyAuthUsername, (proxyAuthPassword==null)?"":proxyAuthPassword, currentHost, (proxyAuthDomain==null)?"":proxyAuthDomain));
           }
+
+          HttpHost proxy = new HttpHost(proxyHost, proxyPort);
+
+          localHttpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy);
+        }
+
+        // Set up authentication to use
+        if (authentication != null)
+        {
+          if (Logging.connectors.isDebugEnabled())
+            Logging.connectors.debug("WEB: For "+myUrl+", discovered matching authentication credentials");
+          localHttpClient.getCredentialsProvider().setCredentials(AuthScope.ANY,
+            authentication.makeCredentialsObject(host));
         }
+          
+        httpClient = localHttpClient;
+      }
 
 
+      // Set the parameters we haven't keyed on (so these can change from request to request)
+      httpClient.getParams().setIntParameter(CoreConnectionPNames.SO_TIMEOUT,socketTimeoutMilliseconds);
+      httpClient.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,connectionTimeoutMilliseconds);
+      httpClient.getParams().setBooleanParameter(ClientPNames.HANDLE_REDIRECTS,redirectOK);
+
+      if (host != null)
+      {
         if (Logging.connectors.isDebugEnabled())
-          Logging.connectors.debug("WEB: Got an HttpClient object after "+new Long(System.currentTimeMillis()-startTime).toString()+" ms.");
+          Logging.connectors.debug("WEB: For "+myUrl+", setting virtual host to "+host);
+        // This will use default port and scheme; it is possible that we should set these explicitly
+        HttpHost virtualHost = new HttpHost(host);
+        httpClient.getParams().setParameter(ClientPNames.VIRTUAL_HOST,virtualHost);
+      }
+
+
+      if (Logging.connectors.isDebugEnabled())
+        Logging.connectors.debug("WEB: Got an HttpClient object after "+new Long(System.currentTimeMillis()-startTime).toString()+" ms.");
 
-        startFetchTime = System.currentTimeMillis();
+      startFetchTime = System.currentTimeMillis();
 
-        int pageFetchMethod = FormData.SUBMITMETHOD_GET;
+      int pageFetchMethod = FormData.SUBMITMETHOD_GET;
+      if (formData != null)
+        pageFetchMethod = formData.getSubmitMethod();
+      switch (pageFetchMethod)
+      {
+      case FormData.SUBMITMETHOD_GET:
+        // MUST be just the path, or apparently we wind up resetting the HostConfiguration
+        // Add additional parameters to url path
+        String fullUrlPath;
         if (formData != null)
-          pageFetchMethod = formData.getSubmitMethod();
-        switch (pageFetchMethod)
         {
-        case FormData.SUBMITMETHOD_GET:
-          // MUST be just the path, or apparently we wind up resetting the HostConfiguration
-          // Add additional parameters to url path
-          String fullUrlPath;
-          if (formData != null)
-          {
-            StringBuilder psb = new StringBuilder(urlPath);
-            Iterator iter = formData.getElementIterator();
-            char appendChar;
-            if (urlPath.indexOf("?") == -1)
-              appendChar = '?';
-            else
-              appendChar = '&';
+          StringBuilder psb = new StringBuilder(urlPath);
+          Iterator iter = formData.getElementIterator();
+          char appendChar;
+          if (urlPath.indexOf("?") == -1)
+            appendChar = '?';
+          else
+            appendChar = '&';
+          try
+          {
             while (iter.hasNext())
             {
-              FormDataElement e = (FormDataElement)iter.next();
+              FormDataElement el = (FormDataElement)iter.next();
               psb.append(appendChar);
               appendChar = '&';
-              String param = e.getElementName();
-              String value = e.getElementValue();
+              String param = el.getElementName();
+              String value = el.getElementValue();
               psb.append(java.net.URLEncoder.encode(param,"utf-8"));
               if (value != null)
+              {
                 psb.append('=').append(java.net.URLEncoder.encode(value,"utf-8"));
+              }
             }
-            fullUrlPath = psb.toString();
           }
-          else
-          {
-            fullUrlPath = urlPath;
-          }
-          // Hack; apparently httpclient treats // as a protocol specifier and so it rips off the first section of the path in that case.
-          while (fullUrlPath.startsWith("//"))
-            fullUrlPath = fullUrlPath.substring(1);
-          if (Logging.connectors.isDebugEnabled())
-            Logging.connectors.debug("WEB: Get method for '"+fullUrlPath+"'");
-          fetchMethod = new GetMethod(fullUrlPath);
-          break;
-        case FormData.SUBMITMETHOD_POST:
-          if (Logging.connectors.isDebugEnabled())
-            Logging.connectors.debug("WEB: Post method for '"+urlPath+"'");
-          // MUST be just the path, or apparently we wind up resetting the HostConfiguration
-          PostMethod postMethod = new PostMethod(urlPath);
-          // Add parameters to post variables
-          if (formData != null)
+          catch (java.io.UnsupportedEncodingException e)
           {
-            Iterator iter = formData.getElementIterator();
-            while (iter.hasNext())
-            {
-              FormDataElement e = (FormDataElement)iter.next();
-              String param = e.getElementName();
-              String value = e.getElementValue();
-              if (Logging.connectors.isDebugEnabled())
-                Logging.connectors.debug("WEB: Post parameter name '"+param+"' value '"+value+"' for '"+urlPath+"'");
-              postMethod.addParameter(param,value);
-            }
+            throw new ManifoldCFException("Unsupported encoding: "+e.getMessage(),e);
           }
-          fetchMethod = postMethod;
-          break;
-        default:
-          throw new ManifoldCFException("Illegal method type: "+Integer.toString(pageFetchMethod));
-        }
 
-        // Set all appropriate headers and parameters
-        fetchMethod.setRequestHeader("User-Agent",userAgent);
-        fetchMethod.setRequestHeader("From",from);
-        HttpMethodParams params = fetchMethod.getParams();
-        if (host != null)
+          fullUrlPath = psb.toString();
+        }
+        else
         {
-          if (Logging.connectors.isDebugEnabled())
-            Logging.connectors.debug("WEB: For "+myUrl+", setting virtual host to "+host);
-          params.setVirtualHost(host);
+          fullUrlPath = urlPath;
         }
-        params.setSoTimeout(socketTimeoutMilliseconds);
-        params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY_MEDIUM_SECURITY);
-        params.setParameter(HttpMethodParams.SINGLE_COOKIE_HEADER,new Boolean(true));
-        fetchMethod.setParams(params);
-        fetchMethod.setFollowRedirects(redirectOK);
-
-        // Clear all current cookies
-        HttpState state = client.getState();
-        state.clearCookies();
+        // Hack; apparently httpclient treats // as a protocol specifier and so it rips off the first section of the path in that case.
+        while (fullUrlPath.startsWith("//"))
+          fullUrlPath = fullUrlPath.substring(1);
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("WEB: Get method for '"+fullUrlPath+"'");
+        fetchMethod = new HttpGet(fullUrlPath);
+        break;
+      case FormData.SUBMITMETHOD_POST:
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("WEB: Post method for '"+urlPath+"'");
+        // MUST be just the path, or apparently we wind up resetting the HostConfiguration
+        HttpPost postMethod = new HttpPost(urlPath);
+        List<NameValuePair> nvps = new ArrayList<NameValuePair>();
 
-        // If we have any cookies to set, set them.
-        if (loginCookies != null)
+        // Add parameters to post variables
+        if (formData != null)
         {
-          if (Logging.connectors.isDebugEnabled())
-            Logging.connectors.debug("WEB: Adding "+Integer.toString(loginCookies.getCookieCount())+" cookies for '"+urlPath+"'");
-          int h = 0;
-          while (h < loginCookies.getCookieCount())
+          Iterator iter = formData.getElementIterator();
+          while (iter.hasNext())
           {
-            state.addCookie(loginCookies.getCookie(h++));
+            FormDataElement e = (FormDataElement)iter.next();
+            String param = e.getElementName();
+            String value = e.getElementValue();
+            if (Logging.connectors.isDebugEnabled())
+              Logging.connectors.debug("WEB: Post parameter name '"+param+"' value '"+value+"' for '"+urlPath+"'");
+            nvps.add(new BasicNameValuePair(param,value));
           }
         }
+        try
+        {
+          postMethod.setEntity(new UrlEncodedFormEntity(nvps,HTTP.UTF_8));
+        }
+        catch (java.io.UnsupportedEncodingException e)
+        {
+          throw new ManifoldCFException("Unsupported UTF-8 encoding: "+e.getMessage(),e);
+        }
+        fetchMethod = postMethod;
+        break;
+      default:
+        throw new ManifoldCFException("Illegal method type: "+Integer.toString(pageFetchMethod));
+      }
 
-        // Copy out the current cookies, in case the fetch fails
-        lastFetchCookies = loginCookies;
+      // Set all appropriate headers and parameters
+      fetchMethod.setHeader(new BasicHeader("User-Agent",userAgent));
+      fetchMethod.setHeader(new BasicHeader("From",from));
+        
+      // Clear all current cookies
+      httpClient.getCookieStore().clear();
 
-        // Set up authentication to use
-        if (authentication != null)
+      // If we have any cookies to set, set them.
+      if (loginCookies != null)
+      {
+        if (Logging.connectors.isDebugEnabled())
+          Logging.connectors.debug("WEB: Adding "+Integer.toString(loginCookies.getCookieCount())+" cookies for '"+urlPath+"'");
+        int h = 0;
+        while (h < loginCookies.getCookieCount())
         {
-          if (Logging.connectors.isDebugEnabled())
-            Logging.connectors.debug("WEB: For "+myUrl+", discovered matching authentication credentials");
-          state.setCredentials(AuthScope.ANY, authentication.makeCredentialsObject(host));
+          httpClient.getCookieStore().addCookie(loginCookies.getCookie(h++));
         }
+      }
 
-        // Set the state.  May not be necessary, but I'd rather not depend on undocumented httpclient implementation details.
-        client.setState(state);
+      // Copy out the current cookies, in case the fetch fails
+      lastFetchCookies = loginCookies;
 
-        // Fire it off!
+      // Create the thread
+      methodThread = new ExecuteMethodThread(this, httpClient, fetchMethod);
+      try
+      {
+        methodThread.start();
+        threadStarted = true;
         try
         {
-          ExecuteMethodThread t = new ExecuteMethodThread(client,clientConf,fetchMethod);
-          try
-          {
-            t.start();
-            t.join();
-            Throwable thr = t.getException();
-            if (thr != null)
-            {
-              throw thr;
-            }
-            statusCode = t.getResponse();
-            if (recordEverything)
-              dataSession.setResponseCode(statusCode);
-          }
-          catch (InterruptedException e)
-          {
-            t.interrupt();
-            // We need the caller to abandon any connections left around, so rethrow in a way that forces them to process the event properly.
-            throw e;
-          }
-
-          // At least we didn't get an exception!  Copy out the current cookies for later reference.
-          lastFetchCookies = new CookieSet(client.getState().getCookies());
-
+          statusCode = methodThread.getResponseCode();
+            
           switch (statusCode)
           {
           case HttpStatus.SC_REQUEST_TIMEOUT:
@@ -1616,112 +1500,85 @@ public class ThrottledFetcher
           default:
             return;
           }
-
-        }
-        catch (java.net.SocketTimeoutException e)
-        {
-          throwable = e;
-          long currentTime = System.currentTimeMillis();
-          throw new ServiceInterruption("Timed out waiting for IO for '"+myUrl+"': "+e.getMessage(), e, currentTime + TIME_5MIN,
-            currentTime + TIME_2HRS,-1,false);
-        }
-        catch (org.apache.commons.httpclient.ConnectTimeoutException e)
-        {
-          throwable = e;
-          long currentTime = System.currentTimeMillis();
-          throw new ServiceInterruption("Timed out waiting for connection for '"+myUrl+"': "+e.getMessage(), e, currentTime + TIME_5MIN,
-            currentTime + TIME_2HRS,-1,false);
-        }
-        catch (InterruptedIOException e)
-        {
-          //Logging.connectors.warn("IO interruption seen",e);
-          throwable = new ManifoldCFException("Interrupted: "+e.getMessage(),e);
-          statusCode = FETCH_INTERRUPTED;
-          throw new ManifoldCFException("Interrupted",ManifoldCFException.INTERRUPTED);
-        }
-        catch (org.apache.commons.httpclient.RedirectException e)
-        {
-          throwable = e;
-          statusCode = FETCH_CIRCULAR_REDIRECT;
-          if (recordEverything)
-            dataSession.setResponseCode(statusCode);
-          return;
-        }
-        catch (org.apache.commons.httpclient.NoHttpResponseException e)
-        {
-          throwable = e;
-          long currentTime = System.currentTimeMillis();
-          throw new ServiceInterruption("Timed out waiting for response for '"+myUrl+"': "+e.getMessage(), e, currentTime + TIME_15MIN,
-            currentTime + TIME_2HRS,-1,false);
-        }
-        catch (java.net.ConnectException e)
-        {
-          throwable = e;
-          long currentTime = System.currentTimeMillis();
-          throw new ServiceInterruption("Timed out waiting for a connection for '"+myUrl+"': "+e.getMessage(), e, currentTime + TIME_2HRS,
-            currentTime + TIME_6HRS,-1,false);
-        }
-        catch (javax.net.ssl.SSLException e)
-        {
-          // Probably this is an incorrectly configured trust store
-          throwable = new ManifoldCFException("SSL handshake error: "+e.getMessage()+"; check your connection's Certificate configuration",e);
-          statusCode = FETCH_IO_ERROR;
-          if (recordEverything)
-            dataSession.setResponseCode(statusCode);
-          return;
         }
-        catch (IOException e)
+        catch (InterruptedException e)
         {
-          // Treat this as a bad url.  We don't know what happened, but it isn't something we are going to naively
-          // retry on.
-          throwable = e;
-          statusCode = FETCH_IO_ERROR;
-          if (recordEverything)
-            dataSession.setResponseCode(statusCode);
-          return;
+          methodThread.interrupt();
+          methodThread = null;
+          threadStarted = false;
+          throw e;
         }
 
       }
       catch (InterruptedException e)
       {
-        // Drop the current connection, and in fact the whole pool, on the floor.
+        // Drop the current connection on the floor, so it cannot be reused.
         fetchMethod = null;
-        connManager = null;
         throwable = new ManifoldCFException("Interrupted: "+e.getMessage(),e);
         statusCode = FETCH_INTERRUPTED;
         throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
       }
-      catch (IllegalArgumentException e)
+      catch (java.net.SocketTimeoutException e)
       {
-        throwable = new ManifoldCFException("Illegal URI: '"+myUrl+"'",e);
-        statusCode = FETCH_BAD_URI;
-        if (recordEverything)
-          dataSession.setResponseCode(statusCode);
-        return;
+        throwable = e;
+        long currentTime = System.currentTimeMillis();
+        throw new ServiceInterruption("Timed out waiting for IO for '"+myUrl+"': "+e.getMessage(), e, currentTime + TIME_5MIN,
+          currentTime + TIME_2HRS,-1,false);
+      }
+      catch (ConnectTimeoutException e)
+      {
+        throwable = e;
+        long currentTime = System.currentTimeMillis();
+        throw new ServiceInterruption("Timed out waiting for connection for '"+myUrl+"': "+e.getMessage(), e, currentTime + TIME_5MIN,
+          currentTime + TIME_2HRS,-1,false);
+      }
+      catch (InterruptedIOException e)
+      {
+        //Logging.connectors.warn("IO interruption seen",e);
+        throwable = new ManifoldCFException("Interrupted: "+e.getMessage(),e);
+        statusCode = FETCH_INTERRUPTED;
+        throw new ManifoldCFException("Interrupted",ManifoldCFException.INTERRUPTED);
       }
-      catch (IllegalStateException e)
+      catch (RedirectException e)
       {
-        throwable = new ManifoldCFException("Illegal state while fetching URI: '"+myUrl+"'",e);
-        statusCode = FETCH_SEQUENCE_ERROR;
-        if (recordEverything)
-          dataSession.setResponseCode(statusCode);
+        throwable = e;
+        statusCode = FETCH_CIRCULAR_REDIRECT;
         return;
       }
-      catch (ServiceInterruption e)
+      catch (NoHttpResponseException e)
       {
-        throw e;
+        throwable = e;
+        long currentTime = System.currentTimeMillis();
+        throw new ServiceInterruption("Timed out waiting for response for '"+myUrl+"': "+e.getMessage(), e, currentTime + TIME_15MIN,
+          currentTime + TIME_2HRS,-1,false);
       }
-      catch (ManifoldCFException e)
+      catch (java.net.ConnectException e)
       {
-        throw e;
+        throwable = e;
+        long currentTime = System.currentTimeMillis();
+        throw new ServiceInterruption("Timed out waiting for a connection for '"+myUrl+"': "+e.getMessage(), e, currentTime + TIME_2HRS,
+          currentTime + TIME_6HRS,-1,false);
+      }
+      catch (javax.net.ssl.SSLException e)
+      {
+        // Probably this is an incorrectly configured trust store
+        throwable = new ManifoldCFException("SSL handshake error: "+e.getMessage()+"; check your connection's Certificate configuration",e);
+        statusCode = FETCH_IO_ERROR;
+        return;
+      }
+      catch (IOException e)
+      {
+        // Treat this as a bad url.  We don't know what happened, but it isn't something we are going to naively
+        // retry on.
+        throwable = e;
+        statusCode = FETCH_IO_ERROR;
+        return;
       }
       catch (Throwable e)
       {
         Logging.connectors.debug("WEB: Caught an unexpected exception: "+e.getMessage(),e);
         throwable = e;
         statusCode = FETCH_UNKNOWN_ERROR;
-        if (recordEverything)
-          dataSession.setResponseCode(statusCode);
         return;
       }
 
@@ -1744,7 +1601,28 @@ public class ThrottledFetcher
     public LoginCookies getLastFetchCookies()
       throws ManifoldCFException, ServiceInterruption
     {
-      return lastFetchCookies;
+      if (fetchMethod == null)
+        throw new ManifoldCFException("Attempt to get cookies when there is no method");
+      if (methodThread == null || threadStarted == false)
+        throw new ManifoldCFException("Attempt to get cookies when no method thread");
+      try
+      {
+        return methodThread.getCookies();
+      }
+      catch (InterruptedException e)
+      {
+        methodThread.interrupt();
+        throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+      }
+      catch (HttpException e)
+      {
+        handleHTTPException(e,"reading cookies");
+      }
+      catch (IOException e)
+      {
+        handleIOException(e,"reading cookies");
+      }
+      return null;
     }
 
     /** Get response headers
@@ -1754,23 +1632,28 @@ public class ThrottledFetcher
     public Map<String,List<String>> getResponseHeaders()
       throws ManifoldCFException, ServiceInterruption
     {
-      Header[] headers = fetchMethod.getResponseHeaders();
-      Map<String,List<String>> rval = new HashMap<String,List<String>>();
-      int i = 0;
-      while (i < headers.length)
+      if (fetchMethod == null)
+        throw new ManifoldCFException("Attempt to get headers when there is no method");
+      if (methodThread == null || threadStarted == false)
+        throw new ManifoldCFException("Attempt to get headers when no method thread");
+      try
       {
-        Header h = headers[i++];
-        String name = h.getName();
-        String value = h.getValue();
-        List<String> values = rval.get(name);
-        if (values == null)
-        {
-          values = new ArrayList<String>();
-          rval.put(name,values);
-        }
-        values.add(value);
+        return methodThread.getResponseHeaders();
       }
-      return rval;
+      catch (InterruptedException e)
+      {
+        methodThread.interrupt();
+        throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+      }
+      catch (HttpException e)
+      {
+        handleHTTPException(e,"reading headers");
+      }
+      catch (IOException e)
+      {
+        handleIOException(e,"reading headers");
+      }
+      return null;
     }
 
     /** Get a specified response header, if it exists.
@@ -1781,12 +1664,28 @@ public class ThrottledFetcher
     public String getResponseHeader(String headerName)
       throws ManifoldCFException, ServiceInterruption
     {
-      Header h = fetchMethod.getResponseHeader(headerName);
-      if (h == null)
-        return null;
-      if (recordEverything)
-        dataSession.addHeader(headerName,h.getValue());
-      return h.getValue();
+      if (fetchMethod == null)
+        throw new ManifoldCFException("Attempt to get a header when there is no method");
+      if (methodThread == null || threadStarted == false)
+        throw new ManifoldCFException("Attempt to get a header when no method thread");
+      try
+      {
+        return methodThread.getFirstHeader(headerName);
+      }
+      catch (InterruptedException e)
+      {
+        methodThread.interrupt();
+        throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+      }
+      catch (HttpException e)
+      {
+        handleHTTPException(e,"reading header");
+      }
+      catch (IOException e)
+      {
+        handleIOException(e,"reading header");
+      }
+      return null;
     }
 
     /** Get the response input stream.  It is the responsibility of the caller
@@ -1797,44 +1696,27 @@ public class ThrottledFetcher
       throws ManifoldCFException, ServiceInterruption
     {
       if (fetchMethod == null)
-        throw new ManifoldCFException("Attempt to get a response when there is no method");
+        throw new ManifoldCFException("Attempt to get an input stream when there is no method");
+      if (methodThread == null || threadStarted == false)
+        throw new ManifoldCFException("Attempt to get an input stream when no method thread");
       try
       {
-        if (recordEverything)
-          dataSession.endHeader();
-        InputStream bodyStream = fetchMethod.getResponseBodyAsStream();
-        if (bodyStream == null)
-        {
-          Logging.connectors.debug("Web: Couldn't set up response stream for '"+myUrl+"', retrying");
-          throw new ServiceInterruption("Failed to set up body response stream for "+myUrl,null,TIME_5MIN,-1L,2,false);
-        }
-        return new ThrottledInputstream(this,bodyStream,dataSession);
-      }
-      catch (java.net.SocketTimeoutException e)
-      {
-        Logging.connectors.debug("Web: Socket timeout exception setting up response stream for '"+myUrl+"', retrying");
-        throw new ServiceInterruption("Socket timeout exception setting up response stream: "+e.getMessage(),e,System.currentTimeMillis()+TIME_5MIN,-1L,2,false);
-      }
-      catch (org.apache.commons.httpclient.ConnectTimeoutException e)
-      {
-        Logging.connectors.debug("Web: Connect timeout exception setting up response stream for '"+myUrl+"', retrying");
-        throw new ServiceInterruption("Connect timeout exception setting up response stream: "+e.getMessage(),e,System.currentTimeMillis()+TIME_5MIN,-1L,2,false);
+        return methodThread.getSafeInputStream();
       }
-      catch (InterruptedIOException e)
+      catch (InterruptedException e)
       {
-        //Logging.connectors.warn("IO interruption seen: "+e.getMessage(),e);
+        methodThread.interrupt();
         throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
       }
       catch (IOException e)
       {
-        Logging.connectors.debug("Web: IO exception setting up response stream for '"+myUrl+"', retrying");
-        throw new ServiceInterruption("IO exception setting up response stream: "+e.getMessage(),e,System.currentTimeMillis()+TIME_5MIN,-1L,2,false);
+        handleIOException(e, "reading response stream");
       }
-      catch (IllegalStateException e)
+      catch (HttpException e)
       {
-        Logging.connectors.debug("Web: State error getting response body for '"+myUrl+"', retrying");
-        throw new ServiceInterruption("State error getting response body: "+e.getMessage(),e,TIME_5MIN,-1L,2,false);
+        handleHTTPException(e, "reading response stream");
       }
+      return null;
     }
 
     /** Get limited response as a string.
@@ -1860,31 +1742,11 @@ public class ThrottledFetcher
           is.close();
         }
       }
-      catch (java.net.SocketTimeoutException e)
-      {
-        Logging.connectors.debug("Web: Socket timeout exception reading response stream for '"+myUrl+"', retrying");
-        throw new ServiceInterruption("Socket timeout exception reading response stream: "+e.getMessage(),e,System.currentTimeMillis()+TIME_5MIN,-1L,2,false);
-      }
-      catch (org.apache.commons.httpclient.ConnectTimeoutException e)
-      {
-        Logging.connectors.debug("Web: Connect timeout exception reading response stream for '"+myUrl+"', retrying");
-        throw new ServiceInterruption("Connect timeout exception reading response stream: "+e.getMessage(),e,System.currentTimeMillis()+TIME_5MIN,-1L,2,false);
-      }
-      catch (InterruptedIOException e)
-      {
-        //Logging.connectors.warn("IO interruption seen: "+e.getMessage(),e);
-        throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
-      }
       catch (IOException e)
       {
-        Logging.connectors.debug("Web: IO exception reading response stream for '"+myUrl+"', retrying");
-        throw new ServiceInterruption("IO exception reading response stream: "+e.getMessage(),e,System.currentTimeMillis()+TIME_5MIN,-1L,2,false);
-      }
-      catch (IllegalStateException e)
-      {
-        Logging.connectors.debug("Web: State error reading response body for '"+myUrl+"', retrying");
-        throw new ServiceInterruption("State error reading response body: "+e.getMessage(),e,TIME_5MIN,-1L,2,false);
+        handleIOException(e,"reading limited response");
       }
+      return null;
     }
 
     /** Note that the connection fetch was interrupted by something.
@@ -1908,6 +1770,24 @@ public class ThrottledFetcher
     {
       if (fetchType != null)
       {
+        // Abort the connection, if not already complete
+        try
+        {
+          methodThread.abort();
+        }
+        catch (InterruptedException e)
+        {
+          throw new ManifoldCFException(e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+        }
+        catch (HttpException e)
+        {
+          handleTerminalHTTPException(e,"aborting");
+        }
+        catch (IOException e)
+        {
+          handleTerminalIOException(e,"aborting");
+        }
+
         long endTime = System.currentTimeMillis();
         int i = 0;
         while (i < throttleBinArray.length)
@@ -1932,20 +1812,30 @@ public class ThrottledFetcher
             Logging.connectors.debug("WEB: Fetch exception for '"+myUrl+"'",throwable);
         }
 
-
-        // Clear out all the parameters
-        if (fetchMethod != null)
+        // Shut down (join) the connection thread, if any, and if it started
+        if (threadStarted)
         {
           try
           {
-            fetchMethod.releaseConnection();
+            methodThread.finishUp();
           }
-          catch (IllegalStateException e)
+          catch (InterruptedException e)
+          {
+            throw new ManifoldCFException(e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+          }
+          catch (HttpException e)
           {
-            // looks like the fetch method didn't have one, or it was already released.  Just eat the exception.
+            handleTerminalHTTPException(e,"finishing up");
           }
-          fetchMethod = null;
+          catch (IOException e)
+          {
+            handleTerminalIOException(e,"finishing up");
+          }
+          threadStarted = false;
+          methodThread = null;
         }
+        
+        fetchMethod = null;
         throwable = null;
         startFetchTime = -1L;
         myUrl = null;
@@ -1999,6 +1889,104 @@ public class ThrottledFetcher
         poolLock.notifyAll();
       }
     }
+    
+    protected void handleTerminalHTTPException(HttpException e, String activity)
+      throws ManifoldCFException
+    {
+      long currentTime = System.currentTimeMillis();
+      Logging.connectors.debug("Web: Unexpected HTTP exception "+activity+" for '"+myUrl+"'");
+      throw new ManifoldCFException("Unexpected HTTP exception "+activity+": "+e.getMessage(),e);
+    }
+
+    protected void handleHTTPException(HttpException e, String activity)
+      throws ServiceInterruption, ManifoldCFException
+    {
+      long currentTime = System.currentTimeMillis();
+      Logging.connectors.debug("Web: HTTP exception "+activity+" for '"+myUrl+"', retrying");
+      throw new ServiceInterruption("HTTP exception "+activity+": "+e.getMessage(),e,currentTime+TIME_5MIN,-1L,2,false);
+    }
+
+    protected void handleTerminalIOException(IOException e, String activity)
+      throws ManifoldCFException
+    {
+      if (e instanceof java.net.SocketTimeoutException)
+      {
+        Logging.connectors.debug("Web: Unexpected socket timeout exception "+activity+" for '"+myUrl+"', retrying");
+        throw new ManifoldCFException("Unexpected socket timeout exception "+activity+": "+e.getMessage(),e);
+      }
+      if (e instanceof ConnectTimeoutException)
+      {
+        Logging.connectors.debug("Web: Unexpected connect timeout exception "+activity+" for '"+myUrl+"', retrying");
+        throw new ManifoldCFException("Unexpected connect timeout exception "+activity+": "+e.getMessage(),e);
+      }
+      if (e instanceof InterruptedIOException)
+      {
+        throw new ManifoldCFException("Interrupted",ManifoldCFException.INTERRUPTED);
+      }
+      if (e instanceof NoHttpResponseException)
+      {
+        throw new ManifoldCFException("Unexpectedly timed out "+activity+" for '"+myUrl+"': "+e.getMessage(),e);
+      }
+      if (e instanceof java.net.ConnectException)
+      {
+        throw new ManifoldCFException("Unexpectedly timed out "+activity+" for '"+myUrl+"': "+e.getMessage(),e);
+      }
+      if (e instanceof java.net.NoRouteToHostException)
+      {
+        // This exception means we know the IP address but can't get there.  That's either a firewall issue, or it's something transient
+        // with the network.  Some degree of retry is probably wise.
+        throw new ManifoldCFException("Unexpected no route to host during "+activity+" for '"+myUrl+"': "+e.getMessage(),e);
+      }
+      Logging.connectors.debug("Web: Unexpected IO exception "+activity+" for '"+myUrl+"': "+e.getMessage(),e);
+      throw new ManifoldCFException("Unexpected IO exception "+activity+": "+e.getMessage(),e);
+    }
+    
+    protected void handleIOException(IOException e, String activity)
+      throws ServiceInterruption, ManifoldCFException
+    {
+      if (e instanceof java.net.SocketTimeoutException)
+      {
+        long currentTime = System.currentTimeMillis();
+        Logging.connectors.debug("Web: Socket timeout exception "+activity+" for '"+myUrl+"', retrying");
+        throw new ServiceInterruption("Socket timeout exception "+activity+": "+e.getMessage(),e,currentTime+TIME_5MIN,-1L,2,false);
+      }
+      if (e instanceof ConnectTimeoutException)
+      {
+        long currentTime = System.currentTimeMillis();
+        Logging.connectors.debug("Web: Connect timeout exception "+activity+" for '"+myUrl+"', retrying");
+        throw new ServiceInterruption("Connect timeout exception "+activity+": "+e.getMessage(),e,currentTime+TIME_5MIN,-1L,2,false);
+      }
+      if (e instanceof InterruptedIOException)
+      {
+        methodThread.interrupt();
+        throw new ManifoldCFException("Interrupted",ManifoldCFException.INTERRUPTED);
+      }
+      if (e instanceof NoHttpResponseException)
+      {
+        // Give up after 2 hours.
+        long currentTime = System.currentTimeMillis();
+        throw new ServiceInterruption("Timed out "+activity+" for '"+myUrl+"'", e, currentTime + 15L * 60000L,
+          currentTime + 120L * 60000L,-1,false);
+      }
+      if (e instanceof java.net.ConnectException)
+      {
+        // Give up after 6 hours.
+        long currentTime = System.currentTimeMillis();
+        throw new ServiceInterruption("Timed out "+activity+" for '"+myUrl+"'", e, currentTime + 1000000L,
+          currentTime + 720L * 60000L,-1,false);
+      }
+      if (e instanceof java.net.NoRouteToHostException)
+      {
+        // This exception means we know the IP address but can't get there.  That's either a firewall issue, or it's something transient
+        // with the network.  Some degree of retry is probably wise.
+        long currentTime = System.currentTimeMillis();
+        throw new ServiceInterruption("No route to host during "+activity+" for '"+myUrl+"'", e, currentTime + 1000000L,
+          currentTime + 720L * 60000L,-1,false);
+      }
+      long currentTime = System.currentTimeMillis();
+      Logging.connectors.debug("Web: IO exception "+activity+" for '"+myUrl+"', retrying");
+      throw new ServiceInterruption("IO exception "+activity+": "+e.getMessage(),e,currentTime+TIME_5MIN,-1L,2,false);
+    }
 
   }
 
@@ -2015,15 +2003,12 @@ public class ThrottledFetcher
     /** The stream we are wrapping. */
     protected InputStream inputStream;
 
-    protected DataSession dataSession;
-
     /** Constructor.
     */
-    public ThrottledInputstream(ThrottledConnection connection, InputStream is, DataSession dataSession)
+    public ThrottledInputstream(ThrottledConnection connection, InputStream is)
     {
       this.throttledConnection = connection;
       this.inputStream = is;
-      this.dataSession = dataSession;
     }
 
     /** Read a byte.
@@ -2094,8 +2079,6 @@ public class ThrottledFetcher
         try
         {
           amt = inputStream.read(b,off,len);
-          if (recordEverything && amt != -1)
-            dataSession.write(b,off,amt);
           return amt;
         }
         finally
@@ -2222,14 +2205,10 @@ public class ThrottledFetcher
   }
 
   /** HTTPClient secure socket factory, which implements SecureProtocolSocketFactory
-  */
   protected static class WebSecureSocketFactory implements SecureProtocolSocketFactory
   {
-    /** This is the javax.net socket factory.
-    */
     protected javax.net.ssl.SSLSocketFactory socketFactory;
 
-    /** Constructor */
     public WebSecureSocketFactory(javax.net.ssl.SSLSocketFactory socketFactory)
     {
       this.socketFactory = socketFactory;
@@ -2350,7 +2329,8 @@ public class ThrottledFetcher
     }
 
   }
-
+  */
+  
   /** Create a secure socket in a thread, so that we can "give up" after a while if the socket fails to connect.
   */
   protected static class SocketCreateThread extends Thread
@@ -2405,4 +2385,333 @@ public class ThrottledFetcher
     }
   }
 
+  /** This thread does the actual socket communication with the server.
+  * It's set up so that it can be abandoned at shutdown time.
+  *
+  * The way it works is as follows:
+  * - it starts the transaction
+  * - it receives the response, and saves that for the calling class to inspect
+  * - it transfers the data part to an input stream provided to the calling class
+  * - it shuts the connection down
+  *
+  * If there is an error, the sequence is aborted, and an exception is recorded
+  * for the calling class to examine.
+  *
+  * The calling class basically accepts the sequence above.  It starts the
+  * thread, and tries to get a response code.  If instead an exception is seen,
+  * the exception is thrown up the stack.
+  */
+  protected static class ExecuteMethodThread extends Thread
+  {
+    /** The connection */
+    protected final ThrottledConnection theConnection;
+    /** Client and method, all preconfigured */
+    protected final AbstractHttpClient httpClient;
+    protected final HttpRequestBase executeMethod;
+    
+    protected HttpResponse response = null;
+    protected Throwable responseException = null;
+    protected LoginCookies cookies = null;
+    protected Throwable cookieException = null;
+    protected XThreadInputStream threadStream = null;
+    protected boolean threadCreated = false;
+    protected Throwable streamException = null;
+
+    protected Throwable shutdownException = null;
+
+    protected Throwable generalException = null;
+    
+    public ExecuteMethodThread(ThrottledConnection theConnection,
+      AbstractHttpClient httpClient, HttpRequestBase executeMethod)
+    {
+      super();
+      setDaemon(true);
+      this.theConnection = theConnection;
+      this.httpClient = httpClient;
+      this.executeMethod = executeMethod;
+    }
+
+    public void run()
+    {
+      try
+      {
+        try
+        {
+          // Call the execute method appropriately
+          synchronized (this)
+          {
+            try
+            {
+              response = httpClient.execute(executeMethod);
+            }
+            catch (java.net.SocketTimeoutException e)
+            {
+              responseException = e;
+            }
+            catch (ConnectTimeoutException e)
+            {
+              responseException = e;
+            }
+            catch (InterruptedIOException e)
+            {
+              throw e;
+            }
+            catch (Throwable e)
+            {
+              responseException = e;
+            }
+            this.notifyAll();
+          }
+          
+          // Fetch the cookies
+          if (responseException == null)
+          {
+            synchronized (this)
+            {
+              try
+              {
+                cookies = new CookieSet(httpClient.getCookieStore().getCookies());
+              }
+              catch (Throwable e)
+              {
+                cookieException = e;
+              }
+              this.notifyAll();
+            }
+          }
+
+          // Start the transfer of the content
+          if (cookieException == null && responseException == null)
+          {
+            synchronized (this)
+            {
+              try
+              {
+                InputStream bodyStream = response.getEntity().getContent();
+                if (bodyStream != null)
+                {
+                  bodyStream = new ThrottledInputstream(theConnection,bodyStream);
+                  threadStream = new XThreadInputStream(bodyStream);
+                }
+                threadCreated = true;
+              }
+              catch (java.net.SocketTimeoutException e)
+              {
+                streamException = e;
+              }
+              catch (ConnectTimeoutException e)
+              {
+                streamException = e;
+              }
+              catch (InterruptedIOException e)
+              {
+                throw e;
+              }
+              catch (Throwable e)
+              {
+                streamException = e;
+              }
+              this.notifyAll();
+            }
+          }
+          
+          if (cookieException == null && responseException == null && streamException == null)
+          {
+            if (threadStream != null)
+            {
+              // Stuff the content until we are done
+              threadStream.stuffQueue();
+            }
+          }
+          
+        }
+        finally
+        {
+          synchronized (this)
+          {
+            try
+            {
+              executeMethod.abort();
+            }
+            catch (Throwable e)
+            {
+              shutdownException = e;
+            }
+            this.notifyAll();
+          }
+        }
+      }
+      catch (Throwable e)
+      {
+        // We catch exceptions here that should ONLY be InterruptedExceptions, as a result of the thread being aborted.
+        this.generalException = e;
+      }
+    }
+
+    public int getResponseCode()
+      throws InterruptedException, IOException, HttpException
+    {
+      // Must wait until the response object is there
+      while (true)
+      {
+        synchronized (this)
+        {
+          checkException(responseException);
+          if (response != null)
+            return response.getStatusLine().getStatusCode();
+          wait();
+        }
+      }
+    }
+
+    public Map<String,List<String>> getResponseHeaders()
+      throws InterruptedException, IOException, HttpException
+    {
+      // Must wait for the response object to appear
+      while (true)
+      {
+        synchronized (this)
+        {
+          checkException(responseException);
+          if (response != null)
+          {
+            Header[] headers = response.getAllHeaders();
+            Map<String,List<String>> rval = new HashMap<String,List<String>>();
+            int i = 0;
+            while (i < headers.length)
+            {
+              Header h = headers[i++];
+              String name = h.getName();
+              String value = h.getValue();
+              List<String> values = rval.get(name);
+              if (values == null)
+              {
+                values = new ArrayList<String>();
+                rval.put(name,values);
+              }
+              values.add(value);
+            }
+            return rval;
+          }
+          wait();
+        }
+      }
+
+    }
+    
+    public String getFirstHeader(String headerName)
+      throws InterruptedException, IOException, HttpException
+    {
+      // Must wait for the response object to appear
+      while (true)
+      {
+        synchronized (this)
+        {
+          checkException(responseException);
+          if (response != null)
+          {
+            Header h = response.getFirstHeader(headerName);
+            if (h == null)
+              return null;
+            return h.getValue();
+          }
+          wait();
+        }
+      }
+    }
+
+    public LoginCookies getCookies()
+      throws InterruptedException, IOException, HttpException
+    {
+      while (true)
+      {
+        synchronized (this)
+        {
+          if (responseException != null)
+            throw new IllegalStateException("Check for response before getting cookies");
+          checkException(cookieException);
+          if (cookies != null)
+            return cookies;
+          wait();
+        }
+      }
+    }
+    
+    public InputStream getSafeInputStream()
+      throws InterruptedException, IOException, HttpException
+    {
+      // Must wait until stream is created, or until we note an exception was thrown.
+      while (true)
+      {
+        synchronized (this)
+        {
+          if (responseException != null)
+            throw new IllegalStateException("Check for response before getting stream");
+          if (cookieException != null)
+            throw new IllegalStateException("Check for cookies before getting stream");
+          checkException(streamException);
+          if (threadCreated)
+            return threadStream;
+          wait();
+        }
+      }
+    }
+    
+    public void abort()
+      throws InterruptedException, IOException, HttpException
+    {
+      // This will be called during the stream access, either
+      // in addition to getSafeInputStream or in exchange.
+      // So we wait for the stream, and when we have it we
+      // kill it, and that will cause the whole thread to abort, 
+      // if it isn't already done.
+      while (true)
+      {
+        synchronized (this)
+        {
+          if (responseException != null)
+            throw new IllegalStateException("Check for response before aborting stream");
+          if (cookieException != null)
+            throw new IllegalStateException("Check for cookies before getting stream");
+          checkException(streamException);
+          if (threadCreated)
+          {
+            if (threadStream != null)
+              threadStream.abort();
+            return;
+          }
+          wait();
+        }
+      }
+    }
+    
+    public void finishUp()
+      throws InterruptedException, IOException, HttpException
+    {
+      join();
+      checkException(shutdownException);
+    }
+    
+    protected synchronized void checkException(Throwable exception)
+      throws IOException, HttpException
+    {
+      if (exception != null)
+      {
+        // Throw the current exception, but clear it, so no further throwing is possible on the same problem.
+        Throwable e = exception;
+        if (e instanceof IOException)
+          throw (IOException)e;
+        else if (e instanceof HttpException)
+          throw (HttpException)e;
+        else if (e instanceof RuntimeException)
+          throw (RuntimeException)e;
+        else if (e instanceof Error)
+          throw (Error)e;
+        else
+          throw new RuntimeException("Unhandled exception of type: "+e.getClass().getName(),e);
+      }
+    }
+
+  }
+
 }

Modified: manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java?rev=1411890&r1=1411889&r2=1411890&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java Tue Nov 20 21:27:02 2012
@@ -49,6 +49,7 @@ import org.apache.http.message.BasicName
 import org.apache.http.protocol.HTTP;
 import org.apache.http.util.EntityUtils;
 import org.apache.http.message.BasicHeader;
+import org.apache.http.client.params.ClientPNames;
 
 import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.client.CircularRedirectException;
@@ -174,6 +175,7 @@ public class WikiConnector extends org.a
       params.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,false);
       params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,300000);
       params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,300000);
+      params.setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS,true);
       DefaultHttpClient localHttpClient = new DefaultHttpClient(connectionManager,params);
       httpClient = localHttpClient;
       



Mime
View raw message