manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1410936 - in /manifoldcf/branches/CONNECTORS-120/connectors: rss/ rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ rss/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/rss/ webcrawler/connec...
Date Sun, 18 Nov 2012 17:16:04 GMT
Author: kwright
Date: Sun Nov 18 17:15:55 2012
New Revision: 1410936

URL: http://svn.apache.org/viewvc?rev=1410936&view=rev
Log:
Fix a couple of problems with the RSS connector, and also port the wiki connector to httpcomponents.  Preliminary work only on web connector.

Modified:
    manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/Robots.java
    manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
    manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/rss/common_en_US.properties
    manifoldcf/branches/CONNECTORS-120/connectors/rss/pom.xml
    manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IThrottledConnection.java
    manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
    manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java
    manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
    manifoldcf/branches/CONNECTORS-120/connectors/wiki/pom.xml

Modified: manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/Robots.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/Robots.java?rev=1410936&r1=1410935&r2=1410936&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/Robots.java (original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/Robots.java Sun Nov 18 17:15:55 2012
@@ -511,7 +511,7 @@ public class Robots
       }
       catch (IOException e)
       {
-        throw new ServiceInterruption("Couldn't fetch robots.txt from "+protocol+"://"+hostName+":"+Integer.toString(port),currentTime + 300000L);
+        throw new ServiceInterruption("Couldn't fetch robots.txt from "+protocol+"://"+hostName+((port==-1)?"":":"+Integer.toString(port)),currentTime + 300000L);
       }
       finally
       {

Modified: manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java?rev=1410936&r1=1410935&r2=1410936&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java (original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/rss/ThrottledFetcher.java Sun Nov 18 17:15:55 2012
@@ -217,13 +217,20 @@ public class ThrottledFetcher
   protected static class ThrottledConnection implements IThrottledConnection
   {
     /** The connection bandwidth we want */
-    protected double minimumMillisecondsPerBytePerServer;
+    protected final double minimumMillisecondsPerBytePerServer;
     /** The maximum open connections per server */
-    protected int maxOpenConnectionsPerServer;
+    protected final int maxOpenConnectionsPerServer;
     /** The minimum time between fetches */
-    protected long minimumMillisecondsPerFetchPerServer;
+    protected final long minimumMillisecondsPerFetchPerServer;
     /** The server object we use to track connections and fetches. */
-    protected Server server;
+    protected final Server server;
+    /** Connection timeout in milliseconds */
+    protected final int connectionTimeoutMilliseconds;
+    /** The client connection manager */
+    protected final ClientConnectionManager connectionManager;
+    /** The httpclient */
+    protected final HttpClient httpClient;
+
     /** The method object */
     protected HttpRequestBase executeMethod = null;
     /** The start-fetch time */
@@ -238,17 +245,11 @@ public class ThrottledFetcher
     protected String fetchType = null;
     /** The current bytes in the current fetch */
     protected long fetchCounter = 0L;
-    /** Connection timeout in milliseconds */
-    protected int connectionTimeoutMilliseconds;
 
     /** The thread that is actually doing the work */
     protected ExecuteMethodThread methodThread = null;
     /** Set if thread has been started */
     protected boolean threadStarted = false;
-    /** The client connection manager */
-    protected ClientConnectionManager connectionManager = null;
-    /** The httpclient */
-    protected HttpClient httpClient = null;
     
     /** Constructor.
     */
@@ -293,7 +294,7 @@ public class ThrottledFetcher
 
         HttpHost proxy = new HttpHost(proxyHost, proxyPort);
 
-        httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy);
+        localHttpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy);
       }
       
       httpClient = localHttpClient;

Modified: manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/rss/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/rss/common_en_US.properties?rev=1410936&r1=1410935&r2=1410936&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/rss/common_en_US.properties (original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/rss/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/rss/common_en_US.properties Sun Nov 18 17:15:55 2012
@@ -72,11 +72,11 @@ RSSConnector.FeedConnectTimeout=Feed con
 RSSConnector.DefaultFeedRefetchTime=Default feed refetch time (minutes)
 RSSConnector.MinimumFeedRefetchTime=Minimum feed refetch time (minutes)
 RSSConnector.BadFeedRefetchTime=Bad feed refetch time (minutes)
-RSSConnector.NoDechromedContent= No dechromed content
-RSSConnector.DechromedContentIfPresentInDescriptionField= Dechromed content, if present, in 'description' field
-RSSConnector.Dechromed ContentIfPresentInContentField= Dechromed content, if present, in 'content' field
-RSSConnector.UseChromedContentIfNoDechromedContentFound= Use chromed content if no dechromed content found
-RSSConnector.NeverUseChromedContent= Never use chromed content
+RSSConnector.NoDechromedContent=No dechromed content
+RSSConnector.DechromedContentIfPresentInDescriptionField=Dechromed content, if present, in 'description' field
+RSSConnector.Dechromed ContentIfPresentInContentField=Dechromed content, if present, in 'content' field
+RSSConnector.UseChromedContentIfNoDechromedContentFound=Use chromed content if no dechromed content found
+RSSConnector.NeverUseChromedContent=Never use chromed content
 RSSConnector.DeleteToken=Delete token #
 RSSConnector.AddAccessToken=Add access token
 RSSConnector.DeleteMetadata=Delete metadata #

Modified: manifoldcf/branches/CONNECTORS-120/connectors/rss/pom.xml
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/rss/pom.xml?rev=1410936&r1=1410935&r2=1410936&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/rss/pom.xml (original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/rss/pom.xml Sun Nov 18 17:15:55 2012
@@ -89,9 +89,9 @@
       <version>${project.version}</version>
     </dependency>
     <dependency>
-      <groupId>commons-httpclient</groupId>
-      <artifactId>commons-httpclient</artifactId>
-      <version>${commons-httpclient.version}</version>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+      <version>${httpcomponent.version}</version>
     </dependency>
     <dependency>
       <groupId>commons-logging</groupId>

Modified: manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IThrottledConnection.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IThrottledConnection.java?rev=1410936&r1=1410935&r2=1410936&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IThrottledConnection.java (original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/IThrottledConnection.java Sun Nov 18 17:15:55 2012
@@ -64,8 +64,7 @@ public interface IThrottledConnection
   */
   public void executeFetch(String urlPath, String userAgent, String from, int connectionTimeoutMilliseconds,
     int socketTimeoutMilliseconds, boolean redirectOK, String host, FormData formData,
-    LoginCookies loginCookies,
-    String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)
+    LoginCookies loginCookies)
     throws ManifoldCFException, ServiceInterruption;
 
   /** Get the http response code.

Modified: manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java?rev=1410936&r1=1410935&r2=1410936&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java (original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/ThrottledFetcher.java Sun Nov 18 17:15:55 2012
@@ -110,7 +110,8 @@ public class ThrottledFetcher
     PageCredentials authentication,
     IKeystoreManager trustStore,
     ThrottleDescription throttleDescription, String[] binNames,
-    int connectionLimit)
+    int connectionLimit,
+    String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)
     throws ManifoldCFException
   {
     // First, create a protocol factory object, if we can
@@ -259,7 +260,8 @@ public class ThrottledFetcher
               }
               else
               {
-                connectionToReuse = cb.findConnection(maxConnections,bins,protocol,server,port,authentication,trustStoreString);
+                connectionToReuse = cb.findConnection(maxConnections,bins,protocol,server,port,authentication,trustStoreString,
+                  proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword);
               }
 
               // Increment after we successfully handled this bin
@@ -368,7 +370,8 @@ public class ThrottledFetcher
 
           // If we have a connection located, activate it.
           if (connectionToReuse == null)
-            connectionToReuse = new ThrottledConnection(protocol,server,port,authentication,myFactory,trustStoreString,bins);
+            connectionToReuse = new ThrottledConnection(protocol,server,port,authentication,myFactory,trustStoreString,bins,
+              proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword);
           connectionToReuse.setup(throttleDescription);
           return connectionToReuse;
         }
@@ -517,7 +520,8 @@ public class ThrottledFetcher
     */
     public synchronized ThrottledConnection findConnection(int maxConnections,
       ConnectionBin[] binNames, String protocol, String server, int port,
-      PageCredentials authentication, String trustStoreString)
+      PageCredentials authentication, String trustStoreString,
+      String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)
       throws PoolException
     {
       //sanityCheck();
@@ -559,7 +563,8 @@ public class ThrottledFetcher
       rval.activate();
       //sanityCheck();
 
-      if (!rval.matches(binNames,protocol,server,port,authentication,trustStoreString))
+      if (!rval.matches(binNames,protocol,server,port,authentication,trustStoreString,
+        proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword))
       {
         // Destroy old connection.  That should free up space for a new creation.
         rval.destroy();
@@ -1016,7 +1021,17 @@ public class ThrottledFetcher
     protected long startFetchTime = -1L;
     /** The cookies from the last fetch */
     protected LoginCookies lastFetchCookies = null;
-
+    /** Proxy host */
+    protected final String proxyHost;
+    /** Proxy port */
+    protected final int proxyPort;
+    /** Proxy auth domain */
+    protected final String proxyAuthDomain;
+    /** Proxy auth user name */
+    protected final String proxyAuthUsername;
+    /** Proxy auth password */
+    protected final String proxyAuthPassword;
+    
     /** Protocol socket factory */
     protected ProtocolSocketFactory secureSocketFactory = null;
     protected ProtocolFactory myFactory = null;
@@ -1028,8 +1043,14 @@ public class ThrottledFetcher
     /** Constructor.  Create a connection with a specific server and port, and
     * register it as active against all bins. */
     public ThrottledConnection(String protocol, String server, int port, PageCredentials authentication,
-      ProtocolFactory myFactory, String trustStoreString, ConnectionBin[] connectionBins)
+      ProtocolFactory myFactory, String trustStoreString, ConnectionBin[] connectionBins,
+      String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)
     {
+      this.proxyHost = proxyHost;
+      this.proxyPort = proxyPort;
+      this.proxyAuthDomain = proxyAuthDomain;
+      this.proxyAuthUsername = proxyAuthUsername;
+      this.proxyAuthPassword = proxyAuthPassword;
       this.protocol = protocol;
       this.server = server;
       this.port = port;
@@ -1073,24 +1094,78 @@ public class ThrottledFetcher
 
     /** See if this instances matches a given server and port. */
     public boolean matches(ConnectionBin[] bins, String protocol, String server, int port, PageCredentials authentication,
-      String trustStoreString)
+      String trustStoreString, String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)
     {
-      if (this.trustStoreString == null && trustStoreString != null)
-        return false;
-      if (this.trustStoreString != null && trustStoreString == null)
-        return false;
-      if (this.trustStoreString != null && !this.trustStoreString.equals(trustStoreString))
-        return false;
+      if (this.trustStoreString == null || trustStoreString == null)
+      {
+        if (this.trustStoreString != trustStoreString)
+          return false;
+      }
+      else
+      {
+        if (!this.trustStoreString.equals(trustStoreString))
+          return false;
+      }
 
-      if (this.authentication == null && authentication != null)
-        return false;
-      if (this.authentication != null && authentication == null)
-        return false;
-      if (this.authentication != null && !this.authentication.equals(authentication))
-        return false;
+      if (this.authentication == null || authentication == null)
+      {
+        if (this.authentication != authentication)
+          return false;
+      }
+      else
+      {
+        if (!this.authentication.equals(authentication))
+          return false;
+      }
 
+      if (this.proxyHost == null || proxyHost == null)
+      {
+        if (this.proxyHost != proxyHost)
+          return false;
+      }
+      else
+      {
+        if (!this.proxyHost.equals(proxyHost))
+          return false;
+        if (this.proxyAuthDomain == null || proxyAuthDomain == null)
+        {
+          if (this.proxyAuthDomain != proxyAuthDomain)
+            return false;
+        }
+        else
+        {
+          if (!this.proxyAuthDomain.equals(proxyAuthDomain))
+            return false;
+        }
+        if (this.proxyAuthUsername == null || proxyAuthUsername == null)
+        {
+          if (this.proxyAuthUsername != proxyAuthUsername)
+            return false;
+        }
+        else
+        {
+          if (!this.proxyAuthUsername.equals(proxyAuthUsername))
+            return false;
+        }
+        if (this.proxyAuthPassword == null || proxyAuthPassword == null)
+        {
+          if (this.proxyAuthPassword != proxyAuthPassword)
+            return false;
+        }
+        else
+        {
+          if (!this.proxyAuthPassword.equals(proxyAuthPassword))
+            return false;
+        }
+      }
+      
+      if (this.proxyPort != proxyPort)
+        return false;
+      
+      
       if (this.connectionBinArray.length != bins.length || !this.protocol.equals(protocol) || !this.server.equals(server) || this.port != port)
         return false;
+      
       int i = 0;
       while (i < bins.length)
       {
@@ -1202,6 +1277,7 @@ public class ThrottledFetcher
     * @param fetchType is a short descriptive string describing the kind of fetch being requested.  This
     *        is used solely for logging purposes.
     */
+    @Override
     public void beginFetch(String fetchType)
       throws ManifoldCFException
     {
@@ -1293,10 +1369,10 @@ public class ThrottledFetcher
     * @param formData describes additional form arguments and how to fetch the page.
     * @param loginCookies describes the cookies that should be in effect for this page fetch.
     */
+    @Override
     public void executeFetch(String urlPath, String userAgent, String from, int connectionTimeoutMilliseconds,
       int socketTimeoutMilliseconds, boolean redirectOK, String host, FormData formData,
-      LoginCookies loginCookies,
-      String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)
+      LoginCookies loginCookies)
       throws ManifoldCFException, ServiceInterruption
     {
       StringBuilder sb = new StringBuilder(protocol);
@@ -1366,13 +1442,9 @@ public class ThrottledFetcher
           clientConf.setProxy(proxyHost,proxyPort);
           if (proxyAuthUsername != null && proxyAuthUsername.length() > 0)
           {
-            if (proxyAuthPassword == null)
-              proxyAuthPassword = "";
-            if (proxyAuthDomain == null)
-              proxyAuthDomain = "";
             // Set up NTLM credentials for this fetch too.
             client.getState().setProxyCredentials(AuthScope.ANY,
-              new NTCredentials(proxyAuthUsername,proxyAuthPassword,currentHost,proxyAuthDomain));
+              new NTCredentials(proxyAuthUsername,(proxyAuthPassword==null)?"":proxyAuthPassword,currentHost,(proxyAuthDomain==null)?"":proxyAuthDomain));
           }
         }
 
@@ -1658,6 +1730,7 @@ public class ThrottledFetcher
     /** Get the http response code.
     *@return the response code.  This is either an HTTP response code, or one of the codes above.
     */
+    @Override
     public int getResponseCode()
       throws ManifoldCFException, ServiceInterruption
     {
@@ -1667,6 +1740,7 @@ public class ThrottledFetcher
     /** Get the last fetch cookies.
     *@return the cookies now in effect from the last fetch.
     */
+    @Override
     public LoginCookies getLastFetchCookies()
       throws ManifoldCFException, ServiceInterruption
     {
@@ -1676,6 +1750,7 @@ public class ThrottledFetcher
     /** Get response headers
     *@return a map keyed by header name containing a list of values.
     */
+    @Override
     public Map<String,List<String>> getResponseHeaders()
       throws ManifoldCFException, ServiceInterruption
     {
@@ -1702,6 +1777,7 @@ public class ThrottledFetcher
     *@param headerName is the name of the header.
     *@return the header value, or null if it doesn't exist.
     */
+    @Override
     public String getResponseHeader(String headerName)
       throws ManifoldCFException, ServiceInterruption
     {
@@ -1716,6 +1792,7 @@ public class ThrottledFetcher
     /** Get the response input stream.  It is the responsibility of the caller
     * to close this stream when done.
     */
+    @Override
     public InputStream getResponseBodyStream()
       throws ManifoldCFException, ServiceInterruption
     {
@@ -1762,6 +1839,7 @@ public class ThrottledFetcher
 
     /** Get limited response as a string.
     */
+    @Override
     public String getLimitedResponseBody(int maxSize, String encoding)
       throws ManifoldCFException, ServiceInterruption
     {
@@ -1811,6 +1889,7 @@ public class ThrottledFetcher
 
     /** Note that the connection fetch was interrupted by something.
     */
+    @Override
     public void noteInterrupted(Throwable e)
     {
       if (statusCode > 0)
@@ -1823,6 +1902,7 @@ public class ThrottledFetcher
     /** Done with the fetch.  Call this when the fetch has been completed.  A log entry will be generated
     * describing what was done.
     */
+    @Override
     public void doneFetch(IVersionActivity activities)
       throws ManifoldCFException
     {
@@ -1878,6 +1958,7 @@ public class ThrottledFetcher
 
     /** Close the connection.  Call this to end this server connection.
     */
+    @Override
     public void close()
       throws ManifoldCFException
     {
@@ -1947,6 +2028,7 @@ public class ThrottledFetcher
 
     /** Read a byte.
     */
+    @Override
     public int read()
       throws IOException
     {
@@ -1959,6 +2041,7 @@ public class ThrottledFetcher
 
     /** Read lots of bytes.
     */
+    @Override
     public int read(byte[] b)
       throws IOException
     {
@@ -1967,6 +2050,7 @@ public class ThrottledFetcher
 
     /** Read lots of specific bytes.
     */
+    @Override
     public int read(byte[] b, int off, int len)
       throws IOException
     {
@@ -2035,6 +2119,7 @@ public class ThrottledFetcher
 
     /** Skip
     */
+    @Override
     public long skip(long n)
       throws IOException
     {
@@ -2044,6 +2129,7 @@ public class ThrottledFetcher
 
     /** Get available.
     */
+    @Override
     public int available()
       throws IOException
     {
@@ -2052,6 +2138,7 @@ public class ThrottledFetcher
 
     /** Mark.
     */
+    @Override
     public void mark(int readLimit)
     {
       inputStream.mark(readLimit);
@@ -2059,6 +2146,7 @@ public class ThrottledFetcher
 
     /** Reset.
     */
+    @Override
     public void reset()
       throws IOException
     {
@@ -2067,6 +2155,7 @@ public class ThrottledFetcher
 
     /** Check if mark is supported.
     */
+    @Override
     public boolean markSupported()
     {
       return inputStream.markSupported();
@@ -2074,6 +2163,7 @@ public class ThrottledFetcher
 
     /** Close.
     */
+    @Override
     public void close()
       throws IOException
     {

Modified: manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java?rev=1410936&r1=1410935&r2=1410936&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/WebcrawlerConnector.java Sun Nov 18 17:15:55 2012
@@ -710,7 +710,8 @@ public class WebcrawlerConnector extends
                   // Prepare to perform the fetch, and decide what to do with the document.
                   //
                   IThrottledConnection connection = ThrottledFetcher.getConnection(protocol,ipAddress,port,
-                    credential,trustStore,throttleDescription,binNames,connectionLimit);
+                    credential,trustStore,throttleDescription,binNames,connectionLimit,
+                    proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword);
                   try
                   {
                     connection.beginFetch((sessionState == SESSIONSTATE_LOGIN)?FETCH_LOGIN:FETCH_STANDARD);
@@ -719,8 +720,7 @@ public class WebcrawlerConnector extends
 
                       // Execute the fetch!
                       connection.executeFetch(url.getFile(),userAgent,from,connectionTimeoutMilliseconds,
-                        socketTimeoutMilliseconds,false,hostName,formData,lc,
-                        proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword);
+                        socketTimeoutMilliseconds,false,hostName,formData,lc);
                       int response = connection.getResponseCode();
 
                       if (response == 200 || response == 302 || response == 301)
@@ -5050,14 +5050,14 @@ public class WebcrawlerConnector extends
       try
       {
         IThrottledConnection connection = ThrottledFetcher.getConnection(protocol,hostIPAddress,port,credential,
-          trustStore,throttleDescription,binNames,connectionLimit);
+          trustStore,throttleDescription,binNames,connectionLimit,
+          proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword);
         try
         {
           connection.beginFetch(FETCH_ROBOTS);
           try
           {
-            connection.executeFetch("/robots.txt",userAgent,from,connectionTimeoutMilliseconds,socketTimeoutMilliseconds,true,hostName,null,null,
-              proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword);
+            connection.executeFetch("/robots.txt",userAgent,from,connectionTimeoutMilliseconds,socketTimeoutMilliseconds,true,hostName,null,null);
             long expirationTime = currentTime+1000*60*60*24;
             int code = connection.getResponseCode();
             if (code == 200)

Modified: manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java?rev=1410936&r1=1410935&r2=1410936&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/WikiConnector.java Sun Nov 18 17:15:55 2012
@@ -31,11 +31,28 @@ import org.apache.manifoldcf.agents.comm
 import org.apache.manifoldcf.agents.common.XMLStringContext;
 import org.apache.manifoldcf.agents.common.XMLFileContext;
 
-import org.apache.commons.httpclient.*;
-import org.apache.commons.httpclient.methods.*;
-import org.apache.commons.httpclient.params.*;
-import org.apache.commons.httpclient.auth.*;
-import org.apache.commons.httpclient.protocol.*;
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.client.HttpClient;
+import org.apache.http.impl.conn.PoolingClientConnectionManager;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.HttpResponse;
+import org.apache.http.params.BasicHttpParams;
+import org.apache.http.params.HttpParams;
+import org.apache.http.params.CoreConnectionPNames;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.NameValuePair;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.protocol.HTTP;
+import org.apache.http.util.EntityUtils;
+
+import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.http.client.CircularRedirectException;
+import org.apache.http.NoHttpResponseException;
+import org.apache.http.HttpException;
 
 import java.util.*;
 import java.io.*;
@@ -77,7 +94,7 @@ public class WikiConnector extends org.a
   protected String serverDomain = null;
   
   /** Connection management */
-  protected MultiThreadedHttpConnectionManager connectionManager = null;
+  protected ClientConnectionManager connectionManager = null;
 
   protected HttpClient httpClient = null;
   
@@ -136,11 +153,18 @@ public class WikiConnector extends org.a
       baseURL = protocol + "://" + server + ((portString!=null)?":" + portString:"") + path + "/api.php?format=xml&";
 
       // Set up connection manager
-      connectionManager = new MultiThreadedHttpConnectionManager();
-      connectionManager.getParams().setMaxTotalConnections(1);
-
-      httpClient = new HttpClient(connectionManager);
-
+      PoolingClientConnectionManager localConnectionManager = new PoolingClientConnectionManager();
+      localConnectionManager.setMaxTotal(1);
+      connectionManager = localConnectionManager;
+
+      BasicHttpParams params = new BasicHttpParams();
+      params.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY,true);
+      params.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK,false);
+      params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT,300000);
+      params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,300000);
+      DefaultHttpClient localHttpClient = new DefaultHttpClient(connectionManager,params);
+      httpClient = localHttpClient;
+      
       loginToAPI();
       
       hasBeenSetup = true;
@@ -158,7 +182,7 @@ public class WikiConnector extends org.a
       return false;
 
     // Grab the httpclient, and use the same one throughout.
-    HttpClient client = getInitializedClient();
+    HttpClient client = httpClient;
     
     // First step in login process: get the token
     Map<String, String> loginParams = new HashMap<String, String>();
@@ -175,31 +199,14 @@ public class WikiConnector extends org.a
 
     APILoginResult result = new APILoginResult();
         
-    HttpMethodBase method = getInitializedPostMethod(loginURL,loginParams);
-
     try {
+      HttpRequestBase method = getInitializedPostMethod(loginURL,loginParams);
       ExecuteAPILoginThread t = new ExecuteAPILoginThread(client, method, result);
       try {
         t.start();
         t.join();
 
-        Throwable thr = t.getException();
-        if (thr != null) {
-          if (thr instanceof ManifoldCFException) {
-            if (((ManifoldCFException) thr).getErrorCode() == ManifoldCFException.INTERRUPTED) {
-              throw new InterruptedException(thr.getMessage());
-            }
-            throw (ManifoldCFException) thr;
-          } else if (thr instanceof ServiceInterruption) {
-            throw (ServiceInterruption) thr;
-          } else if (thr instanceof IOException) {
-            throw (IOException) thr;
-          } else if (thr instanceof RuntimeException) {
-            throw (RuntimeException) thr;
-          } else {
-            throw (Error) thr;
-          }
-        }
+        handleException(t.getException());
       } catch (ManifoldCFException e) {
         t.interrupt();
         throw e;
@@ -209,6 +216,9 @@ public class WikiConnector extends org.a
       } catch (IOException e) {
         t.interrupt();
         throw e;
+      } catch (HttpException e) {
+	t.interrupt();
+	throw e;
       } catch (InterruptedException e) {
         t.interrupt();
         // We need the caller to abandon any connections left around, so rethrow in a way that forces them to process the event properly.
@@ -228,14 +238,8 @@ public class WikiConnector extends org.a
       }
       
     } catch (InterruptedException e) {
-      // Drop the connection on the floor
-      method = null;
       throw new ManifoldCFException("Interrupted: " + e.getMessage(), e, ManifoldCFException.INTERRUPTED);
     } catch (ManifoldCFException e) {
-      if (e.getErrorCode() == ManifoldCFException.INTERRUPTED) // Drop the connection on the floor
-      {
-        method = null;
-      }
       throw e;
     } catch (java.net.SocketTimeoutException e) {
       long currentTime = System.currentTimeMillis();
@@ -243,48 +247,30 @@ public class WikiConnector extends org.a
     } catch (java.net.SocketException e) {
       long currentTime = System.currentTimeMillis();
       throw new ServiceInterruption("Login received a socket error reading from Wiki server: " + e.getMessage(), e, currentTime + 300000L, currentTime + 12L * 60000L, -1, false);
-    } catch (org.apache.commons.httpclient.ConnectTimeoutException e) {
+    } catch (ConnectTimeoutException e) {
       long currentTime = System.currentTimeMillis();
       throw new ServiceInterruption("Login connection timed out reading from Wiki server: " + e.getMessage(), e, currentTime + 300000L, currentTime + 12L * 60000L, -1, false);
     } catch (InterruptedIOException e) {
-      method = null;
       throw new ManifoldCFException("Interrupted: " + e.getMessage(), e, ManifoldCFException.INTERRUPTED);
     } catch (IOException e) {
       throw new ManifoldCFException("Login had an IO failure: " + e.getMessage(), e);
-    } finally {
-      if (method != null) {
-        method.releaseConnection();
-      }
+    } catch (HttpException e) {
+      throw new ManifoldCFException("Login had an Http exception: "+e.getMessage(), e);
     }
 
     // First request is finished.  Fire off the second one.
     
     loginParams.put("lgtoken", token);
     
-    method = getInitializedPostMethod(loginURL,loginParams);
     try {
-      ExecuteTokenAPILoginThread t = new ExecuteTokenAPILoginThread(client, method, result);
+      HttpRequestBase method = getInitializedPostMethod(loginURL,loginParams);
+      ExecuteTokenAPILoginThread t = new ExecuteTokenAPILoginThread(httpClient, method, result);
       try {
         t.start();
         t.join();
 
-        Throwable thr = t.getException();
-        if (thr != null) {
-          if (thr instanceof ManifoldCFException) {
-            if (((ManifoldCFException) thr).getErrorCode() == ManifoldCFException.INTERRUPTED) {
-              throw new InterruptedException(thr.getMessage());
-            }
-            throw (ManifoldCFException) thr;
-          } else if (thr instanceof ServiceInterruption) {
-            throw (ServiceInterruption) thr;
-          } else if (thr instanceof IOException) {
-            throw (IOException) thr;
-          } else if (thr instanceof RuntimeException) {
-            throw (RuntimeException) thr;
-          } else {
-            throw (Error) thr;
-          }
-        }
+	handleException(t.getException());
+	
       } catch (ManifoldCFException e) {
         t.interrupt();
         throw e;
@@ -294,6 +280,9 @@ public class WikiConnector extends org.a
       } catch (IOException e) {
         t.interrupt();
         throw e;
+      } catch (HttpException e) {
+	t.interrupt();
+	throw e;
       } catch (InterruptedException e) {
         t.interrupt();
         // We need the caller to abandon any connections left around, so rethrow in a way that forces them to process the event properly.
@@ -302,14 +291,8 @@ public class WikiConnector extends org.a
 
       // Fall through
     } catch (InterruptedException e) {
-      // Drop the connection on the floor
-      method = null;
       throw new ManifoldCFException("Interrupted: " + e.getMessage(), e, ManifoldCFException.INTERRUPTED);
     } catch (ManifoldCFException e) {
-      if (e.getErrorCode() == ManifoldCFException.INTERRUPTED) // Drop the connection on the floor
-      {
-        method = null;
-      }
       throw e;
     } catch (java.net.SocketTimeoutException e) {
       long currentTime = System.currentTimeMillis();
@@ -317,18 +300,15 @@ public class WikiConnector extends org.a
     } catch (java.net.SocketException e) {
       long currentTime = System.currentTimeMillis();
       throw new ServiceInterruption("Login received a socket error reading from Wiki server: " + e.getMessage(), e, currentTime + 300000L, currentTime + 12L * 60000L, -1, false);
-    } catch (org.apache.commons.httpclient.ConnectTimeoutException e) {
+    } catch (ConnectTimeoutException e) {
       long currentTime = System.currentTimeMillis();
       throw new ServiceInterruption("Login connection timed out reading from Wiki server: " + e.getMessage(), e, currentTime + 300000L, currentTime + 12L * 60000L, -1, false);
     } catch (InterruptedIOException e) {
-      method = null;
       throw new ManifoldCFException("Interrupted: " + e.getMessage(), e, ManifoldCFException.INTERRUPTED);
     } catch (IOException e) {
       throw new ManifoldCFException("Login had an IO failure: " + e.getMessage(), e);
-    } finally {
-      if (method != null) {
-        method.releaseConnection();
-      }
+    } catch (HttpException e) {
+      throw new ManifoldCFException("Login had an Http exception: "+e.getMessage(), e);
     }
     
     // Check result
@@ -347,12 +327,12 @@ public class WikiConnector extends org.a
   protected class ExecuteAPILoginThread extends Thread {
 
     protected HttpClient client;
-    protected HttpMethodBase executeMethod;
+    protected HttpRequestBase executeMethod;
     protected APILoginResult result;
     protected Throwable exception = null;
     protected String token = null;
 
-    public ExecuteAPILoginThread(HttpClient client, HttpMethodBase executeMethod, APILoginResult result) {
+    public ExecuteAPILoginThread(HttpClient client, HttpRequestBase executeMethod, APILoginResult result) {
       super();
       setDaemon(true);
       this.client = client;
@@ -363,13 +343,13 @@ public class WikiConnector extends org.a
     public void run() {
       try {
         // Call the execute method appropriately
-        int rval = client.executeMethod(executeMethod);
-        if (rval != 200) {
-          throw new ManifoldCFException("Unexpected response code " + rval + ": " + executeMethod.getResponseBodyAsString());
+	HttpResponse rval = client.execute(executeMethod);
+        if (rval.getStatusLine().getStatusCode() != 200) {
+          throw new ManifoldCFException("Unexpected response code " + rval.getStatusLine().getStatusCode() + ": " + readResponseAsString(rval));
         }
 
         // Read response and make sure it's valid
-        InputStream is = executeMethod.getResponseBodyAsStream();
+        InputStream is = rval.getEntity().getContent();
         try {
           // Parse the document.  This will cause various things to occur, within the instantiated XMLContext class.
           //<api>
@@ -388,6 +368,10 @@ public class WikiConnector extends org.a
               x.parse(is);
               token = c.getToken();
             }
+	    catch (InterruptedIOException e)
+	    {
+	      throw e;
+	    }
             catch (IOException e)
             {
               long time = System.currentTimeMillis();
@@ -405,6 +389,8 @@ public class WikiConnector extends org.a
         }
       } catch (Throwable e) {
         this.exception = e;
+      } finally {
+	executeMethod.abort();
       }
     }
 
@@ -494,11 +480,11 @@ public class WikiConnector extends org.a
   protected class ExecuteTokenAPILoginThread extends Thread {
 
     protected HttpClient client;
-    protected HttpMethodBase executeMethod;
+    protected HttpRequestBase executeMethod;
     protected Throwable exception = null;
     protected APILoginResult result;
 
-    public ExecuteTokenAPILoginThread(HttpClient client, HttpMethodBase executeMethod, APILoginResult result) {
+    public ExecuteTokenAPILoginThread(HttpClient client, HttpRequestBase executeMethod, APILoginResult result) {
       super();
       setDaemon(true);
       this.client = client;
@@ -509,13 +495,13 @@ public class WikiConnector extends org.a
     public void run() {
       try {
         // Call the execute method appropriately
-        int rval = client.executeMethod(executeMethod);
-        if (rval != 200) {
-          throw new ManifoldCFException("Unexpected response code " + rval + ": " + executeMethod.getResponseBodyAsString());
+        HttpResponse rval = client.execute(executeMethod);
+        if (rval.getStatusLine().getStatusCode() != 200) {
+          throw new ManifoldCFException("Unexpected response code " + rval.getStatusLine().getStatusCode() + ": " + readResponseAsString(rval));
         }
 
         // Read response and make sure it's valid
-        InputStream is = executeMethod.getResponseBodyAsStream();
+        InputStream is = rval.getEntity().getContent();
         try {
           // Parse the document.  This will cause various things to occur, within the instantiated XMLContext class.
           //<api>
@@ -550,6 +536,8 @@ public class WikiConnector extends org.a
         }
       } catch (Throwable e) {
         this.exception = e;
+      } finally {
+	executeMethod.abort();
       }
     }
 
@@ -639,8 +627,9 @@ public class WikiConnector extends org.a
   public void poll()
     throws ManifoldCFException
   {
-    if (connectionManager != null)
-      connectionManager.closeIdleConnections(60000L);
+    // MHL
+    //if (connectionManager != null)
+    //  connectionManager.closeIdleConnections(60000L);
   }
 
   /** Close the connection.  Call this before discarding the connection.
@@ -1620,29 +1609,26 @@ public class WikiConnector extends org.a
 
   // Protected static classes and methods
 
-  /** Create and initialize an HttpClient instance */
-  protected HttpClient getInitializedClient()
-    throws ServiceInterruption, ManifoldCFException
+  /** Create and initialize an HttpRequestBase */
+  protected HttpRequestBase getInitializedGetMethod(String URL)
+    throws IOException
   {
-    return httpClient;
-  }
-
-  /** Create and initialize an HttpMethodBase */
-  protected HttpMethodBase getInitializedMethod(String URL)
-  {
-    GetMethod method = new GetMethod(URL);
-    method.getParams().setParameter("http.socket.timeout", new Integer(300000));
-    return method;
+    return new HttpGet(URL);
   }
 
   /** Create an initialize a post method */
-  protected HttpMethodBase getInitializedPostMethod(String URL, Map<String,String> params)
+  protected HttpRequestBase getInitializedPostMethod(String URL, Map<String,String> params)
+    throws IOException
   {
-    PostMethod method = new PostMethod(URL);
+    HttpPost method = new HttpPost(URL);
+    List<NameValuePair> pairs = new ArrayList<NameValuePair>();
+    
     for (String key : params.keySet()) {
-      method.setParameter(key, params.get(key));
+      pairs.add(new BasicNameValuePair(key, params.get(key)));
     }
-    method.getParams().setParameter("http.socket.timeout", new Integer(300000));
+    
+    method.setEntity(new UrlEncodedFormEntity(pairs, HTTP.UTF_8));
+    
     return method;
   }
   
@@ -1657,33 +1643,16 @@ public class WikiConnector extends org.a
     boolean loginAttempted = false;
     while (true)
     {
-      HttpClient client = getInitializedClient();
-      HttpMethodBase executeMethod = getInitializedMethod(getCheckURL());
+      HttpClient client = httpClient;
       try
       {
+	HttpRequestBase executeMethod = getInitializedGetMethod(getCheckURL());
         ExecuteCheckThread t = new ExecuteCheckThread(client,executeMethod);
         try
         {
           t.start();
           t.join();
-          Throwable thr = t.getException();
-          if (thr != null)
-          {
-            if (thr instanceof ManifoldCFException)
-            {
-              if (((ManifoldCFException)thr).getErrorCode() == ManifoldCFException.INTERRUPTED)
-                throw new InterruptedException(thr.getMessage());
-              throw (ManifoldCFException)thr;
-            }
-            else if (thr instanceof ServiceInterruption)
-              throw (ServiceInterruption)thr;
-            else if (thr instanceof IOException)
-              throw (IOException)thr;
-            else if (thr instanceof RuntimeException)
-              throw (RuntimeException)thr;
-            else
-              throw (Error)thr;
-          }
+          handleException(t.getException());
           if (loginAttempted || !t.isLoginRequired())
             return;
         }
@@ -1702,6 +1671,11 @@ public class WikiConnector extends org.a
           t.interrupt();
           throw e;
         }
+	catch (HttpException e)
+	{
+	  t.interrupt();
+	  throw e;
+	}
         catch (InterruptedException e)
         {
           t.interrupt();
@@ -1711,15 +1685,10 @@ public class WikiConnector extends org.a
       }
       catch (InterruptedException e)
       {
-        // Drop the connection on the floor
-        executeMethod = null;
         throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
       }
       catch (ManifoldCFException e)
       {
-        if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
-          // Drop the connection on the floor
-          executeMethod = null;
         throw e;
       }
       catch (java.net.SocketTimeoutException e)
@@ -1732,24 +1701,22 @@ public class WikiConnector extends org.a
         long currentTime = System.currentTimeMillis();
         throw new ServiceInterruption("Fetch test received a socket error reading from Wiki server: "+e.getMessage(),e,currentTime+300000L,currentTime+12L * 60000L,-1,false);
       }
-      catch (org.apache.commons.httpclient.ConnectTimeoutException e)
+      catch (ConnectTimeoutException e)
       {
         long currentTime = System.currentTimeMillis();
         throw new ServiceInterruption("Fetch test connection timed out reading from Wiki server: "+e.getMessage(),e,currentTime+300000L,currentTime+12L * 60000L,-1,false);
       }
       catch (InterruptedIOException e)
       {
-        executeMethod = null;
         throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
       }
       catch (IOException e)
       {
         throw new ManifoldCFException("Fetch test had an IO failure: "+e.getMessage(),e);
       }
-      finally
+      catch (HttpException e)
       {
-        if (executeMethod != null)
-          executeMethod.releaseConnection();
+	throw new ManifoldCFException("Fetch test had Http exception: "+e.getMessage(),e);
       }
       
       if (!loginToAPI())
@@ -1771,11 +1738,11 @@ public class WikiConnector extends org.a
   protected static class ExecuteCheckThread extends Thread
   {
     protected HttpClient client;
-    protected HttpMethodBase executeMethod;
+    protected HttpRequestBase executeMethod;
     protected Throwable exception = null;
     protected boolean loginNeeded = false;
 
-    public ExecuteCheckThread(HttpClient client, HttpMethodBase executeMethod)
+    public ExecuteCheckThread(HttpClient client, HttpRequestBase executeMethod)
     {
       super();
       setDaemon(true);
@@ -1788,11 +1755,11 @@ public class WikiConnector extends org.a
       try
       {
         // Call the execute method appropriately
-        int rval = client.executeMethod(executeMethod);
-        if (rval != 200)
-          throw new ManifoldCFException("Unexpected response code: "+rval);
+        HttpResponse rval = client.execute(executeMethod);
+        if (rval.getStatusLine().getStatusCode() != 200)
+          throw new ManifoldCFException("Unexpected response code: "+rval.getStatusLine().getStatusCode()+": "+readResponseAsString(rval));
         // Read response and make sure it's valid
-        InputStream is = executeMethod.getResponseBodyAsStream();
+        InputStream is = rval.getEntity().getContent();
         try
         {
           loginNeeded = parseCheckResponse(is);
@@ -1813,6 +1780,10 @@ public class WikiConnector extends org.a
       {
         this.exception = e;
       }
+      finally
+      {
+	executeMethod.abort();
+      }
     }
 
     public Throwable getException()
@@ -1997,12 +1968,11 @@ public class WikiConnector extends org.a
     boolean loginAttempted = false;
     while (true)
     {
-      HttpClient client = getInitializedClient();
-      HttpMethodBase executeMethod = getInitializedMethod(getListPagesURL(startPageTitle,namespace,prefix));
       try
       {
+	HttpRequestBase executeMethod = getInitializedGetMethod(getListPagesURL(startPageTitle,namespace,prefix));
         PageBuffer pageBuffer = new PageBuffer();
-        ExecuteListPagesThread t = new ExecuteListPagesThread(client,executeMethod,pageBuffer,startPageTitle);
+        ExecuteListPagesThread t = new ExecuteListPagesThread(httpClient,executeMethod,pageBuffer,startPageTitle);
         try
         {
           t.start();
@@ -2019,24 +1989,7 @@ public class WikiConnector extends org.a
           }
           
           t.join();
-          Throwable thr = t.getException();
-          if (thr != null)
-          {
-            if (thr instanceof ManifoldCFException)
-            {
-              if (((ManifoldCFException)thr).getErrorCode() == ManifoldCFException.INTERRUPTED)
-                throw new InterruptedException(thr.getMessage());
-              throw (ManifoldCFException)thr;
-            }
-            else if (thr instanceof ServiceInterruption)
-              throw (ServiceInterruption)thr;
-            else if (thr instanceof IOException)
-              throw (IOException)thr;
-            else if (thr instanceof RuntimeException)
-              throw (RuntimeException)thr;
-            else
-              throw (Error)thr;
-          }
+          handleException(t.getException());
           if (loginAttempted || !t.isLoginRequired())
             return t.getLastPageTitle();
         }
@@ -2055,6 +2008,11 @@ public class WikiConnector extends org.a
           t.interrupt();
           throw e;
         }
+	catch (HttpException e)
+	{
+	  t.interrupt();
+	  throw e;
+	}
         catch (InterruptedException e)
         {
           t.interrupt();
@@ -2069,15 +2027,11 @@ public class WikiConnector extends org.a
       }
       catch (InterruptedException e)
       {
-        // Drop the connection on the floor
-        executeMethod = null;
         throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
       }
       catch (ManifoldCFException e)
       {
         if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
-          // Drop the connection on the floor
-          executeMethod = null;
         throw e;
       }
       catch (java.net.SocketTimeoutException e)
@@ -2090,24 +2044,22 @@ public class WikiConnector extends org.a
         long currentTime = System.currentTimeMillis();
         throw new ServiceInterruption("ListPages received a socket error reading from Wiki server: "+e.getMessage(),e,currentTime+300000L,currentTime+12L * 60000L,-1,false);
       }
-      catch (org.apache.commons.httpclient.ConnectTimeoutException e)
+      catch (ConnectTimeoutException e)
       {
         long currentTime = System.currentTimeMillis();
         throw new ServiceInterruption("ListPages connection timed out reading from Wiki server: "+e.getMessage(),e,currentTime+300000L,currentTime+12L * 60000L,-1,false);
       }
       catch (InterruptedIOException e)
       {
-        executeMethod = null;
         throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
       }
       catch (IOException e)
       {
         throw new ManifoldCFException("ListPages had an IO failure: "+e.getMessage(),e);
       }
-      finally
+      catch (HttpException e)
       {
-        if (executeMethod != null)
-          executeMethod.releaseConnection();
+	throw new ManifoldCFException("ListPages had an HTTP exception: "+e.getMessage(),e);
       }
       
       if (!loginToAPI())
@@ -2147,14 +2099,14 @@ public class WikiConnector extends org.a
   protected static class ExecuteListPagesThread extends Thread
   {
     protected HttpClient client;
-    protected HttpMethodBase executeMethod;
+    protected HttpRequestBase executeMethod;
     protected Throwable exception = null;
     protected PageBuffer pageBuffer;
     protected String lastPageTitle = null;
     protected String startPageTitle;
     protected boolean loginNeeded = false;
 
-    public ExecuteListPagesThread(HttpClient client, HttpMethodBase executeMethod, PageBuffer pageBuffer, String startPageTitle)
+    public ExecuteListPagesThread(HttpClient client, HttpRequestBase executeMethod, PageBuffer pageBuffer, String startPageTitle)
     {
       super();
       setDaemon(true);
@@ -2169,11 +2121,11 @@ public class WikiConnector extends org.a
       try
       {
         // Call the execute method appropriately
-        int rval = client.executeMethod(executeMethod);
-        if (rval != 200)
-          throw new ManifoldCFException("Unexpected response code: "+rval);
+        HttpResponse rval = client.execute(executeMethod);
+        if (rval.getStatusLine().getStatusCode() != 200)
+          throw new ManifoldCFException("Unexpected response code: "+rval.getStatusLine().getStatusCode()+": "+readResponseAsString(rval));
         // Read response and make sure it's valid
-        InputStream is = executeMethod.getResponseBodyAsStream();
+        InputStream is = rval.getEntity().getContent();
         try
         {
           ReturnString returnString = new ReturnString();
@@ -2199,6 +2151,7 @@ public class WikiConnector extends org.a
       finally
       {
         pageBuffer.signalDone();
+	executeMethod.abort();
       }
     }
 
@@ -2424,33 +2377,15 @@ public class WikiConnector extends org.a
     boolean loginAttempted = false;
     while (true)
     {
-      HttpClient client = getInitializedClient();
-      HttpMethodBase executeMethod = getInitializedMethod(getGetDocURLsURL(documentIdentifiers));
       try
       {
-        ExecuteGetDocURLsThread t = new ExecuteGetDocURLsThread(client,executeMethod,urls);
+	HttpRequestBase executeMethod = getInitializedGetMethod(getGetDocURLsURL(documentIdentifiers));
+        ExecuteGetDocURLsThread t = new ExecuteGetDocURLsThread(httpClient,executeMethod,urls);
         try
         {
           t.start();
           t.join();
-          Throwable thr = t.getException();
-          if (thr != null)
-          {
-            if (thr instanceof ManifoldCFException)
-            {
-              if (((ManifoldCFException)thr).getErrorCode() == ManifoldCFException.INTERRUPTED)
-                throw new InterruptedException(thr.getMessage());
-              throw (ManifoldCFException)thr;
-            }
-            else if (thr instanceof ServiceInterruption)
-              throw (ServiceInterruption)thr;
-            else if (thr instanceof IOException)
-              throw (IOException)thr;
-            else if (thr instanceof RuntimeException)
-              throw (RuntimeException)thr;
-            else
-              throw (Error)thr;
-          }
+          handleException(t.getException());
           if (loginAttempted || !t.isLoginRequired())
             return;
         }
@@ -2469,6 +2404,11 @@ public class WikiConnector extends org.a
           t.interrupt();
           throw e;
         }
+	catch (HttpException e)
+	{
+	  t.interrupt();
+	  throw e;
+	}
         catch (InterruptedException e)
         {
           t.interrupt();
@@ -2478,15 +2418,10 @@ public class WikiConnector extends org.a
       }
       catch (InterruptedException e)
       {
-        // Drop the connection on the floor
-        executeMethod = null;
         throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
       }
       catch (ManifoldCFException e)
       {
-        if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
-          // Drop the connection on the floor
-          executeMethod = null;
         throw e;
       }
       catch (java.net.SocketTimeoutException e)
@@ -2499,24 +2434,22 @@ public class WikiConnector extends org.a
         long currentTime = System.currentTimeMillis();
         throw new ServiceInterruption("URL fetch received a socket error reading from Wiki server: "+e.getMessage(),e,currentTime+300000L,currentTime+12L * 60000L,-1,false);
       }
-      catch (org.apache.commons.httpclient.ConnectTimeoutException e)
+      catch (ConnectTimeoutException e)
       {
         long currentTime = System.currentTimeMillis();
         throw new ServiceInterruption("URL fetch connection timed out reading from Wiki server: "+e.getMessage(),e,currentTime+300000L,currentTime+12L * 60000L,-1,false);
       }
       catch (InterruptedIOException e)
       {
-        executeMethod = null;
         throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
       }
       catch (IOException e)
       {
         throw new ManifoldCFException("URL fetch had an IO failure: "+e.getMessage(),e);
       }
-      finally
+      catch (HttpException e)
       {
-        if (executeMethod != null)
-          executeMethod.releaseConnection();
+	throw new ManifoldCFException("URL fetch had an HTTP exception: "+e.getMessage(),e);
       }
       
       if (!loginToAPI())
@@ -2551,12 +2484,12 @@ public class WikiConnector extends org.a
   protected static class ExecuteGetDocURLsThread extends Thread
   {
     protected HttpClient client;
-    protected HttpMethodBase executeMethod;
+    protected HttpRequestBase executeMethod;
     protected Throwable exception = null;
     protected Map<String,String> urls;
     protected boolean loginNeeded = false;
 
-    public ExecuteGetDocURLsThread(HttpClient client, HttpMethodBase executeMethod, Map<String,String> urls)
+    public ExecuteGetDocURLsThread(HttpClient client, HttpRequestBase executeMethod, Map<String,String> urls)
     {
       super();
       setDaemon(true);
@@ -2570,11 +2503,11 @@ public class WikiConnector extends org.a
       try
       {
         // Call the execute method appropriately
-        int rval = client.executeMethod(executeMethod);
-        if (rval != 200)
-          throw new ManifoldCFException("Unexpected response code: "+rval);
+        HttpResponse rval = client.execute(executeMethod);
+        if (rval.getStatusLine().getStatusCode() != 200)
+          throw new ManifoldCFException("Unexpected response code: "+rval.getStatusLine().getStatusCode()+": "+readResponseAsString(rval));
         // Read response and make sure it's valid
-        InputStream is = executeMethod.getResponseBodyAsStream();
+        InputStream is = rval.getEntity().getContent();
         try
         {
           loginNeeded = parseGetDocURLsResponse(is,urls);
@@ -2595,6 +2528,10 @@ public class WikiConnector extends org.a
       {
         this.exception = e;
       }
+      finally
+      {
+	executeMethod.abort();
+      }
     }
 
     public Throwable getException()
@@ -2758,33 +2695,15 @@ public class WikiConnector extends org.a
     boolean loginAttempted = false;
     while (true)
     {
-      HttpClient client = getInitializedClient();
-      HttpMethodBase executeMethod = getInitializedMethod(getGetTimestampURL(documentIdentifiers));
       try
       {
-        ExecuteGetTimestampThread t = new ExecuteGetTimestampThread(client,executeMethod,versions);
+	HttpRequestBase executeMethod = getInitializedGetMethod(getGetTimestampURL(documentIdentifiers));
+        ExecuteGetTimestampThread t = new ExecuteGetTimestampThread(httpClient,executeMethod,versions);
         try
         {
           t.start();
           t.join();
-          Throwable thr = t.getException();
-          if (thr != null)
-          {
-            if (thr instanceof ManifoldCFException)
-            {
-              if (((ManifoldCFException)thr).getErrorCode() == ManifoldCFException.INTERRUPTED)
-                throw new InterruptedException(thr.getMessage());
-              throw (ManifoldCFException)thr;
-            }
-            else if (thr instanceof ServiceInterruption)
-              throw (ServiceInterruption)thr;
-            else if (thr instanceof IOException)
-              throw (IOException)thr;
-            else if (thr instanceof RuntimeException)
-              throw (RuntimeException)thr;
-            else
-              throw (Error)thr;
-          }
+          handleException(t.getException());
           if (loginAttempted || !t.isLoginRequired())
             return;
         }
@@ -2803,6 +2722,11 @@ public class WikiConnector extends org.a
           t.interrupt();
           throw e;
         }
+	catch (HttpException e)
+	{
+	  t.interrupt();
+	  throw e;
+	}
         catch (InterruptedException e)
         {
           t.interrupt();
@@ -2812,15 +2736,10 @@ public class WikiConnector extends org.a
       }
       catch (InterruptedException e)
       {
-        // Drop the connection on the floor
-        executeMethod = null;
         throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
       }
       catch (ManifoldCFException e)
       {
-        if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
-          // Drop the connection on the floor
-          executeMethod = null;
         throw e;
       }
       catch (java.net.SocketTimeoutException e)
@@ -2833,24 +2752,22 @@ public class WikiConnector extends org.a
         long currentTime = System.currentTimeMillis();
         throw new ServiceInterruption("Version fetch received a socket error reading from Wiki server: "+e.getMessage(),e,currentTime+300000L,currentTime+12L * 60000L,-1,false);
       }
-      catch (org.apache.commons.httpclient.ConnectTimeoutException e)
+      catch (ConnectTimeoutException e)
       {
         long currentTime = System.currentTimeMillis();
         throw new ServiceInterruption("Version fetch connection timed out reading from Wiki server: "+e.getMessage(),e,currentTime+300000L,currentTime+12L * 60000L,-1,false);
       }
       catch (InterruptedIOException e)
       {
-        executeMethod = null;
         throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
       }
       catch (IOException e)
       {
         throw new ManifoldCFException("Version fetch had an IO failure: "+e.getMessage(),e);
       }
-      finally
+      catch (HttpException e)
       {
-        if (executeMethod != null)
-          executeMethod.releaseConnection();
+	throw new ManifoldCFException("Version fetch had an HTTP exception: "+e.getMessage(),e);
       }
       
       if (!loginToAPI())
@@ -2885,12 +2802,12 @@ public class WikiConnector extends org.a
   protected static class ExecuteGetTimestampThread extends Thread
   {
     protected HttpClient client;
-    protected HttpMethodBase executeMethod;
+    protected HttpRequestBase executeMethod;
     protected Throwable exception = null;
     protected Map<String,String> versions;
     protected boolean loginNeeded = false;
 
-    public ExecuteGetTimestampThread(HttpClient client, HttpMethodBase executeMethod, Map<String,String> versions)
+    public ExecuteGetTimestampThread(HttpClient client, HttpRequestBase executeMethod, Map<String,String> versions)
     {
       super();
       setDaemon(true);
@@ -2904,11 +2821,11 @@ public class WikiConnector extends org.a
       try
       {
         // Call the execute method appropriately
-        int rval = client.executeMethod(executeMethod);
-        if (rval != 200)
-          throw new ManifoldCFException("Unexpected response code: "+rval);
+        HttpResponse rval = client.execute(executeMethod);
+        if (rval.getStatusLine().getStatusCode() != 200)
+          throw new ManifoldCFException("Unexpected response code: "+rval.getStatusLine().getStatusCode()+": "+readResponseAsString(rval));
         // Read response and make sure it's valid
-        InputStream is = executeMethod.getResponseBodyAsStream();
+        InputStream is = rval.getEntity().getContent();
         try
         {
           loginNeeded = parseGetTimestampResponse(is,versions);
@@ -2929,6 +2846,10 @@ public class WikiConnector extends org.a
       {
         this.exception = e;
       }
+      finally
+      {
+	executeMethod.abort();
+      }
     }
 
     public Throwable getException()
@@ -3163,35 +3084,16 @@ public class WikiConnector extends org.a
     boolean loginAttempted = false;
     while (true)
     {
-      HttpClient client = getInitializedClient();
-      HttpMethodBase executeMethod = getInitializedMethod(getGetNamespacesURL());
-      
       try
       {
-        ExecuteGetNamespacesThread t = new ExecuteGetNamespacesThread(client,executeMethod,namespaces);
+	HttpRequestBase executeMethod = getInitializedGetMethod(getGetNamespacesURL());
+        ExecuteGetNamespacesThread t = new ExecuteGetNamespacesThread(httpClient,executeMethod,namespaces);
         try
         {
           t.start();
           t.join();
           
-          Throwable thr = t.getException();
-          if (thr != null)
-          {
-            if (thr instanceof ManifoldCFException)
-            {
-              if (((ManifoldCFException)thr).getErrorCode() == ManifoldCFException.INTERRUPTED)
-                throw new InterruptedException(thr.getMessage());
-              throw (ManifoldCFException)thr;
-            }
-            else if (thr instanceof ServiceInterruption)
-              throw (ServiceInterruption)thr;
-            else if (thr instanceof IOException)
-              throw (IOException)thr;
-            else if (thr instanceof RuntimeException)
-              throw (RuntimeException)thr;
-            else
-              throw (Error)thr;
-          }
+          handleException(t.getException());
           if (loginAttempted || !t.isLoginRequired())
             return;
         }
@@ -3210,6 +3112,11 @@ public class WikiConnector extends org.a
           t.interrupt();
           throw e;
         }
+	catch (HttpException e)
+	{
+	  t.interrupt();
+	  throw e;
+	}
         catch (InterruptedException e)
         {
           t.interrupt();
@@ -3219,15 +3126,10 @@ public class WikiConnector extends org.a
       }
       catch (InterruptedException e)
       {
-        // Drop the connection on the floor
-        executeMethod = null;
         throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
       }
       catch (ManifoldCFException e)
       {
-        if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
-          // Drop the connection on the floor
-          executeMethod = null;
         throw e;
       }
       catch (java.net.SocketTimeoutException e)
@@ -3240,24 +3142,22 @@ public class WikiConnector extends org.a
         long currentTime = System.currentTimeMillis();
         throw new ServiceInterruption("Get namespaces received a socket error reading from Wiki server: "+e.getMessage(),e,currentTime+300000L,currentTime+12L * 60000L,-1,false);
       }
-      catch (org.apache.commons.httpclient.ConnectTimeoutException e)
+      catch (ConnectTimeoutException e)
       {
         long currentTime = System.currentTimeMillis();
         throw new ServiceInterruption("Get namespaces connection timed out reading from Wiki server: "+e.getMessage(),e,currentTime+300000L,currentTime+12L * 60000L,-1,false);
       }
       catch (InterruptedIOException e)
       {
-        executeMethod = null;
         throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
       }
       catch (IOException e)
       {
         throw new ManifoldCFException("Get namespaces had an IO failure: "+e.getMessage(),e);
       }
-      finally
+      catch (HttpException e)
       {
-        if (executeMethod != null)
-          executeMethod.releaseConnection();
+	throw new ManifoldCFException("Get namespaces had an HTTP exception: "+e.getMessage(),e);
       }
       
       if (!loginToAPI())
@@ -3270,12 +3170,12 @@ public class WikiConnector extends org.a
   protected static class ExecuteGetNamespacesThread extends Thread
   {
     protected HttpClient client;
-    protected HttpMethodBase executeMethod;
+    protected HttpRequestBase executeMethod;
     protected Throwable exception = null;
     protected Map<String,String> namespaces;
     protected boolean loginNeeded = false;
 
-    public ExecuteGetNamespacesThread(HttpClient client, HttpMethodBase executeMethod, Map<String,String> namespaces)
+    public ExecuteGetNamespacesThread(HttpClient client, HttpRequestBase executeMethod, Map<String,String> namespaces)
     {
       super();
       setDaemon(true);
@@ -3289,14 +3189,14 @@ public class WikiConnector extends org.a
       try
       {
         // Call the execute method appropriately
-        int rval = client.executeMethod(executeMethod);
-        if (rval != 200)
+        HttpResponse rval = client.execute(executeMethod);
+        if (rval.getStatusLine().getStatusCode() != 200)
         {
-          throw new ManifoldCFException("Unexpected response code "+rval+": "+executeMethod.getResponseBodyAsString());
+          throw new ManifoldCFException("Unexpected response code "+rval.getStatusLine().getStatusCode()+": "+readResponseAsString(rval));
         }
 
         // Read response and make sure it's valid
-        InputStream is = executeMethod.getResponseBodyAsStream();
+        InputStream is = rval.getEntity().getContent();
         try
         {
           // Parse the document.  This will cause various things to occur, within the instantiated XMLContext class.
@@ -3350,6 +3250,10 @@ public class WikiConnector extends org.a
       {
         this.exception = e;
       }
+      finally
+      {
+	executeMethod.abort();
+      }
     }
 
     public Throwable getException()
@@ -3515,9 +3419,6 @@ public class WikiConnector extends org.a
     boolean loginAttempted = false;
     while (true)
     {
-      HttpClient client = getInitializedClient();
-      HttpMethodBase executeMethod = getInitializedMethod(getGetDocInfoURL(documentIdentifier));
-      
       String statusCode = "UNKNOWN";
       String errorMessage = null;
       long startTime = System.currentTimeMillis();
@@ -3525,7 +3426,8 @@ public class WikiConnector extends org.a
       
       try
       {
-        ExecuteGetDocInfoThread t = new ExecuteGetDocInfoThread(client,executeMethod,documentIdentifier);
+	HttpRequestBase executeMethod = getInitializedGetMethod(getGetDocInfoURL(documentIdentifier));
+        ExecuteGetDocInfoThread t = new ExecuteGetDocInfoThread(httpClient,executeMethod,documentIdentifier);
         try
         {
           t.start();
@@ -3534,24 +3436,7 @@ public class WikiConnector extends org.a
           statusCode = t.getStatusCode();
           errorMessage = t.getErrorMessage();
             
-          Throwable thr = t.getException();
-          if (thr != null)
-          {
-            if (thr instanceof ManifoldCFException)
-            {
-              if (((ManifoldCFException)thr).getErrorCode() == ManifoldCFException.INTERRUPTED)
-                throw new InterruptedException(thr.getMessage());
-              throw (ManifoldCFException)thr;
-            }
-            else if (thr instanceof ServiceInterruption)
-              throw (ServiceInterruption)thr;
-            else if (thr instanceof IOException)
-              throw (IOException)thr;
-            else if (thr instanceof RuntimeException)
-              throw (RuntimeException)thr;
-            else
-              throw (Error)thr;
-          }
+          handleException(t.getException());
    
           // Fetch all the data we need from the thread, and do the indexing.
           File contentFile = t.getContentFile();
@@ -3620,6 +3505,11 @@ public class WikiConnector extends org.a
           t.interrupt();
           throw e;
         }
+	catch (HttpException e)
+	{
+	  t.interrupt();
+	  throw e;
+	}
         catch (InterruptedException e)
         {
           t.interrupt();
@@ -3634,7 +3524,6 @@ public class WikiConnector extends org.a
       catch (InterruptedException e)
       {
         // Drop the connection on the floor
-        executeMethod = null;
         statusCode = null;
         throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
       }
@@ -3642,8 +3531,6 @@ public class WikiConnector extends org.a
       {
         if (e.getErrorCode() == ManifoldCFException.INTERRUPTED)
         {
-          // Drop the connection on the floor
-          executeMethod = null;
           statusCode = null;
         }
         throw e;
@@ -3658,14 +3545,13 @@ public class WikiConnector extends org.a
         long currentTime = System.currentTimeMillis();
         throw new ServiceInterruption("Get doc info received a socket error reading from Wiki server: "+e.getMessage(),e,currentTime+300000L,currentTime+12L * 60000L,-1,false);
       }
-      catch (org.apache.commons.httpclient.ConnectTimeoutException e)
+      catch (ConnectTimeoutException e)
       {
         long currentTime = System.currentTimeMillis();
         throw new ServiceInterruption("Get doc info connection timed out reading from Wiki server: "+e.getMessage(),e,currentTime+300000L,currentTime+12L * 60000L,-1,false);
       }
       catch (InterruptedIOException e)
       {
-        executeMethod = null;
         statusCode = null;
         throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
       }
@@ -3673,10 +3559,12 @@ public class WikiConnector extends org.a
       {
         throw new ManifoldCFException("Get doc info had an IO failure: "+e.getMessage(),e);
       }
+      catch (HttpException e)
+      {
+	throw new ManifoldCFException("Get doc info had an HTTP exception: "+e.getMessage(),e);
+      }
       finally
       {
-        if (executeMethod != null)
-          executeMethod.releaseConnection();
         if (statusCode != null)
           activities.recordActivity(new Long(startTime),ACTIVITY_FETCH,new Long(dataSize),documentIdentifier,statusCode,errorMessage,null);
       }
@@ -3691,7 +3579,7 @@ public class WikiConnector extends org.a
   protected static class ExecuteGetDocInfoThread extends Thread
   {
     protected HttpClient client;
-    protected HttpMethodBase executeMethod;
+    protected HttpRequestBase executeMethod;
     protected Throwable exception = null;
     protected String documentIdentifier;
     protected File contentFile = null;
@@ -3704,7 +3592,7 @@ public class WikiConnector extends org.a
     protected String errorMessage = null;
     protected boolean loginNeeded = false;
 
-    public ExecuteGetDocInfoThread(HttpClient client, HttpMethodBase executeMethod, String documentIdentifier)
+    public ExecuteGetDocInfoThread(HttpClient client, HttpRequestBase executeMethod, String documentIdentifier)
     {
       super();
       setDaemon(true);
@@ -3718,14 +3606,14 @@ public class WikiConnector extends org.a
       try
       {
         // Call the execute method appropriately
-        int rval = client.executeMethod(executeMethod);
-        if (rval != 200)
+        HttpResponse rval = client.execute(executeMethod);
+        if (rval.getStatusLine().getStatusCode() != 200)
         {
-          statusCode = "HTTP code "+rval;
-          throw new ManifoldCFException("Unexpected response code "+rval+": "+executeMethod.getResponseBodyAsString());
+          statusCode = "HTTP code "+rval.getStatusLine().getStatusCode();
+          throw new ManifoldCFException("Unexpected response code "+rval.getStatusLine().getStatusCode()+": "+readResponseAsString(rval));
         }
         // Read response and make sure it's valid
-        InputStream is = executeMethod.getResponseBodyAsStream();
+        InputStream is = rval.getEntity().getContent();
         try
         {
           // Parse the document.  This will cause various things to occur, within the instantiated XMLContext class.
@@ -3757,6 +3645,10 @@ public class WikiConnector extends org.a
               statusCode = "OK";
               loginNeeded = c.isLoginRequired();
             }
+	    catch (InterruptedIOException e)
+	    {
+	      throw e;
+	    }
             catch (IOException e)
             {
               long time = System.currentTimeMillis();
@@ -3786,6 +3678,10 @@ public class WikiConnector extends org.a
         errorMessage = e.getMessage();
         this.exception = e;
       }
+      finally
+      {
+	executeMethod.abort();
+      }
     }
 
     public Throwable getException()
@@ -4334,4 +4230,66 @@ public class WikiConnector extends org.a
     
   }
   
+  protected static String readResponseAsString(HttpResponse httpResponse)
+    throws IOException
+  {
+    HttpEntity entity = httpResponse.getEntity();
+    if (entity != null)
+    {
+      InputStream is = entity.getContent();
+      try
+      {
+        String charSet = EntityUtils.getContentCharSet(entity);
+        if (charSet == null)
+          charSet = "utf-8";
+        char[] buffer = new char[65536];
+        Reader r = new InputStreamReader(is,charSet);
+        Writer w = new StringWriter();
+        try
+        {
+          while (true)
+          {
+            int amt = r.read(buffer);
+            if (amt == -1)
+              break;
+            w.write(buffer,0,amt);
+          }
+        }
+        finally
+        {
+          w.flush();
+        }
+        return w.toString();
+      }
+      finally
+      {
+        is.close();
+      }
+    }
+    return "";
+  }
+  
+  protected static void handleException(Throwable thr)
+    throws InterruptedException, ManifoldCFException, ServiceInterruption, IOException, HttpException
+  {
+    if (thr != null) {
+      if (thr instanceof ManifoldCFException) {
+	if (((ManifoldCFException) thr).getErrorCode() == ManifoldCFException.INTERRUPTED) {
+	  throw new InterruptedException(thr.getMessage());
+	}
+	throw (ManifoldCFException) thr;
+      } else if (thr instanceof ServiceInterruption) {
+	throw (ServiceInterruption) thr;
+      } else if (thr instanceof IOException) {
+	throw (IOException) thr;
+      } else if (thr instanceof HttpException) {
+	throw (HttpException) thr;
+      } else if (thr instanceof RuntimeException) {
+	throw (RuntimeException) thr;
+      } else {
+	throw (Error) thr;
+      }
+    }
+  }
+
 }

Modified: manifoldcf/branches/CONNECTORS-120/connectors/wiki/pom.xml
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-120/connectors/wiki/pom.xml?rev=1410936&r1=1410935&r2=1410936&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-120/connectors/wiki/pom.xml (original)
+++ manifoldcf/branches/CONNECTORS-120/connectors/wiki/pom.xml Sun Nov 18 17:15:55 2012
@@ -89,9 +89,9 @@
       <version>${project.version}</version>
     </dependency>
     <dependency>
-      <groupId>commons-httpclient</groupId>
-      <artifactId>commons-httpclient</artifactId>
-      <version>${commons-httpclient.version}</version>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+      <version>${httpcomponent.version}</version>
     </dependency>
     <dependency>
       <groupId>commons-logging</groupId>



Mime
View raw message