manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1417060 [3/4] - in /manifoldcf/trunk: ./ connectors/livelink/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/livelink/ connectors/meridio/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/meridio/ connectors...
Date Tue, 04 Dec 2012 17:47:30 GMT
Modified: manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/CommonsHTTPSender.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/CommonsHTTPSender.java?rev=1417060&r1=1417059&r2=1417060&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/CommonsHTTPSender.java
(original)
+++ manifoldcf/trunk/connectors/sharepoint/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharepoint/CommonsHTTPSender.java
Tue Dec  4 17:47:28 2012
@@ -17,6 +17,8 @@
 */
 package org.apache.manifoldcf.crawler.connectors.sharepoint;
 
+import org.apache.manifoldcf.core.common.XThreadInputStream;
+
 import org.apache.axis.AxisFault;
 import org.apache.axis.Constants;
 import org.apache.axis.Message;
@@ -47,6 +49,12 @@ import org.apache.http.ProtocolVersion;
 import org.apache.http.util.EntityUtils;
 import org.apache.http.message.BasicHeader;
 
+import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.http.client.RedirectException;
+import org.apache.http.client.CircularRedirectException;
+import org.apache.http.NoHttpResponseException;
+import org.apache.http.HttpException;
+
 import org.apache.commons.logging.Log;
 
 import javax.xml.soap.MimeHeader;
@@ -69,7 +77,9 @@ import java.net.URL;
 import java.util.ArrayList;
 import java.util.Hashtable;
 import java.util.Iterator;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.List;
 
 /* Class to use httpcomponents to communicate with a SOAP server.
 * I've replaced the original rather complicated class with a much simpler one that
@@ -90,165 +100,6 @@ public class CommonsHTTPSender extends B
     this.clientProperties = CommonsHTTPClientPropertiesFactory.create();
   }
 
-  protected static class ExecuteMethodThread extends Thread
-  {
-    protected final HttpClient httpClient;
-    protected final String targetURL;
-    protected final MessageContext msgContext;
-
-    protected Throwable exception = null;
-    protected int returnCode = 0;
-
-    public ExecuteMethodThread( HttpClient httpClient, String targetURL, MessageContext msgContext
)
-    {
-      super();
-      setDaemon(true);
-      this.httpClient = httpClient;
-      this.targetURL = targetURL;
-      this.msgContext = msgContext;
-    }
-
-    public void run()
-    {
-      try
-      {
-        boolean posting = true;
-        // If we're SOAP 1.2, allow the web method to be set from the
-        // MessageContext.
-        if (msgContext.getSOAPConstants() == SOAPConstants.SOAP12_CONSTANTS) {
-          String webMethod = msgContext.getStrProp(SOAP12Constants.PROP_WEBMETHOD);
-          if (webMethod != null) {
-            posting = webMethod.equals(HTTPConstants.HEADER_POST);
-          }
-        }
-
-        boolean http10 = false;
-        String httpVersion = msgContext.getStrProp(MessageContext.HTTP_TRANSPORT_VERSION);
-        if (httpVersion != null) {
-          if (httpVersion.equals(HTTPConstants.HEADER_PROTOCOL_V10)) {
-            http10 = true;
-          }
-          // assume 1.1
-        }
-
-        HttpRequestBase method;
-        if (posting) {
-          HttpPost postMethod = new HttpPost(targetURL);
-          
-          // set false as default, addContetInfo can overwrite
-          HttpProtocolParams.setUseExpectContinue(postMethod.getParams(),false);
-
-          Message reqMessage = msgContext.getRequestMessage();
-          
-          boolean httpChunkStream = addContextInfo(postMethod, msgContext);
-
-          HttpEntity requestEntity = null;
-          requestEntity = new MessageRequestEntity(reqMessage, httpChunkStream,
-            http10 || !httpChunkStream);
-          postMethod.setEntity(requestEntity);
-          method = postMethod;
-        } else {
-          method = new HttpGet(targetURL);
-        }
-        
-        if (http10)
-          HttpProtocolParams.setVersion(method.getParams(),new ProtocolVersion("HTTP",1,0));
-
-        // Try block to insure that the connection gets cleaned up
-        try
-        {
-          // Begin the fetch
-          HttpResponse response = httpClient.execute(method);
-
-          returnCode = response.getStatusLine().getStatusCode();
-          
-          String contentType =
-            getHeader(response, HTTPConstants.HEADER_CONTENT_TYPE);
-          String contentLocation =
-            getHeader(response, HTTPConstants.HEADER_CONTENT_LOCATION);
-          String contentLength =
-            getHeader(response, HTTPConstants.HEADER_CONTENT_LENGTH);
-
-          if ((returnCode > 199) && (returnCode < 300)) {
-
-            // SOAP return is OK - so fall through
-          } else if (msgContext.getSOAPConstants() ==
-            SOAPConstants.SOAP12_CONSTANTS) {
-            // For now, if we're SOAP 1.2, fall through, since the range of
-            // valid result codes is much greater
-          } else if ((contentType != null) && !contentType.equals("text/html")
-            && ((returnCode > 499) && (returnCode < 600))) {
-
-            // SOAP Fault should be in here - so fall through
-          } else {
-            String statusMessage = response.getStatusLine().toString();
-            AxisFault fault = new AxisFault("HTTP",
-              "(" + returnCode + ")"
-            + statusMessage, null,
-              null);
-
-            fault.setFaultDetailString(
-              Messages.getMessage("return01",
-              "" + returnCode,
-              getResponseBodyAsString(response)));
-            fault.addFaultDetail(Constants.QNAME_FAULTDETAIL_HTTPERRORCODE,
-              Integer.toString(returnCode));
-            throw fault;
-          }
-
-          // Transfer to a temporary file.  If we stream it, we may wind up waiting on the
socket outside this thread.
-          InputStream releaseConnectionOnCloseStream = new FileBackedInputStream(response.getEntity().getContent());
-
-          Header contentEncoding =
-            response.getFirstHeader(HTTPConstants.HEADER_CONTENT_ENCODING);
-          if (contentEncoding != null) {
-            AxisFault fault = new AxisFault("HTTP",
-              "unsupported content-encoding of '"
-            + contentEncoding.getValue()
-            + "' found", null, null);
-            throw fault;
-          }
-
-          Message outMsg = new Message(releaseConnectionOnCloseStream,
-            false, contentType, contentLocation);
-          
-          // Transfer HTTP headers of HTTP message to MIME headers of SOAP message
-          Header[] responseHeaders = response.getAllHeaders();
-          MimeHeaders responseMimeHeaders = outMsg.getMimeHeaders();
-          for (int i = 0; i < responseHeaders.length; i++) {
-            Header responseHeader = responseHeaders[i];
-            responseMimeHeaders.addHeader(responseHeader.getName(),
-              responseHeader.getValue());
-          }
-          outMsg.setMessageType(Message.RESPONSE);
-          
-          // Put the message in the message context.
-          msgContext.setResponseMessage(outMsg);
-        }
-        finally
-        {
-          // Consumes and closes the stream, releasing the connection
-          method.abort();
-        }
-
-      }
-      catch (Throwable e)
-      {
-        this.exception = e;
-      }
-    }
-
-    public Throwable getException()
-    {
-      return exception;
-    }
-
-    public int getResponse()
-    {
-      return returnCode;
-    }
-  }
-
   /**
   * invoke creates a socket connection, sends the request SOAP message and then
   * reads the response SOAP message back from the SOAP server
@@ -274,40 +125,131 @@ public class CommonsHTTPSender extends B
       // Get the HttpClient
       HttpClient httpClient = (HttpClient)msgContext.getProperty(SPSProxyHelper.HTTPCLIENT_PROPERTY);
 
-      ExecuteMethodThread t = new ExecuteMethodThread(httpClient,targetURL.toString(),msgContext);
+      boolean posting = true;
+      // If we're SOAP 1.2, allow the web method to be set from the
+      // MessageContext.
+      if (msgContext.getSOAPConstants() == SOAPConstants.SOAP12_CONSTANTS) {
+        String webMethod = msgContext.getStrProp(SOAP12Constants.PROP_WEBMETHOD);
+        if (webMethod != null) {
+          posting = webMethod.equals(HTTPConstants.HEADER_POST);
+        }
+      }
+
+      boolean http10 = false;
+      String httpVersion = msgContext.getStrProp(MessageContext.HTTP_TRANSPORT_VERSION);
+      if (httpVersion != null) {
+        if (httpVersion.equals(HTTPConstants.HEADER_PROTOCOL_V10)) {
+          http10 = true;
+        }
+        // assume 1.1
+      }
+
+      HttpRequestBase method;
+        
+      if (posting) {
+        HttpPost postMethod = new HttpPost(targetURL.toString());
+          
+        // set false as default, addContetInfo can overwrite
+        HttpProtocolParams.setUseExpectContinue(postMethod.getParams(),false);
+
+        Message reqMessage = msgContext.getRequestMessage();
+          
+        boolean httpChunkStream = addContextInfo(postMethod, msgContext);
+
+        HttpEntity requestEntity = null;
+        requestEntity = new MessageRequestEntity(reqMessage, httpChunkStream,
+          http10 || !httpChunkStream);
+        postMethod.setEntity(requestEntity);
+        method = postMethod;
+      } else {
+        method = new HttpGet(targetURL.toString());
+      }
+        
+      if (http10)
+        HttpProtocolParams.setVersion(method.getParams(),new ProtocolVersion("HTTP",1,0));
+
+      BackgroundHTTPThread methodThread = new BackgroundHTTPThread(httpClient,method);
+      methodThread.start();
       try
       {
-        t.start();
-        t.join();
-        Throwable thr = t.getException();
-        if (thr != null)
+        int returnCode = methodThread.getResponseCode();
+          
+        String contentType =
+          getHeader(methodThread, HTTPConstants.HEADER_CONTENT_TYPE);
+        String contentLocation =
+          getHeader(methodThread, HTTPConstants.HEADER_CONTENT_LOCATION);
+        String contentLength =
+          getHeader(methodThread, HTTPConstants.HEADER_CONTENT_LENGTH);
+        
+        if ((returnCode > 199) && (returnCode < 300)) {
+
+          // SOAP return is OK - so fall through
+        } else if (msgContext.getSOAPConstants() ==
+          SOAPConstants.SOAP12_CONSTANTS) {
+          // For now, if we're SOAP 1.2, fall through, since the range of
+          // valid result codes is much greater
+        } else if ((contentType != null) && !contentType.equals("text/html")
+          && ((returnCode > 499) && (returnCode < 600))) {
+
+          // SOAP Fault should be in here - so fall through
+        } else {
+          String statusMessage = methodThread.getResponseStatus();
+          AxisFault fault = new AxisFault("HTTP",
+            "(" + returnCode + ")"
+          + statusMessage, null,
+            null);
+
+          fault.setFaultDetailString(
+            Messages.getMessage("return01",
+            "" + returnCode,
+            getResponseBodyAsString(methodThread)));
+          fault.addFaultDetail(Constants.QNAME_FAULTDETAIL_HTTPERRORCODE,
+            Integer.toString(returnCode));
+          throw fault;
+        }
+
+        String contentEncoding =
+         methodThread.getFirstHeader(HTTPConstants.HEADER_CONTENT_ENCODING);
+        if (contentEncoding != null) {
+          AxisFault fault = new AxisFault("HTTP",
+            "unsupported content-encoding of '"
+          + contentEncoding
+          + "' found", null, null);
+          throw fault;
+        }
+
+        Map<String,List<String>> responseHeaders = methodThread.getResponseHeaders();
+
+        InputStream dataStream = methodThread.getSafeInputStream();
+
+        Message outMsg = new Message(new BackgroundInputStream(methodThread,dataStream),
+          false, contentType, contentLocation);
+          
+        // Transfer HTTP headers of HTTP message to MIME headers of SOAP message
+        MimeHeaders responseMimeHeaders = outMsg.getMimeHeaders();
+        for (String name : responseHeaders.keySet())
         {
-          if (thr instanceof RuntimeException)
-            throw (RuntimeException)thr;
-          else if (thr instanceof Exception)
-            throw (Exception)thr;
-          else
-            throw (Error)thr;
+          List<String> values = responseHeaders.get(name);
+          for (String value : values) {
+            responseMimeHeaders.addHeader(name,value);
+          }
         }
+        outMsg.setMessageType(Message.RESPONSE);
+          
+        // Put the message in the message context.
+        msgContext.setResponseMessage(outMsg);
+        
+        // Pass off the method thread to the stream for closure
+        methodThread = null;
       }
-      catch (InterruptedException e)
+      finally
       {
-        t.interrupt();
-        throw e;
-      }
-
-      /*
-      if (log.isDebugEnabled()) {
-        if (null == contentLength) {
-          log.debug("\n"
-            + Messages.getMessage("no00", "Content-Length"));
-          }
-          log.debug("\n" + Messages.getMessage("xmlRecd00"));
-          log.debug("-----------------------------------------------");
-          log.debug(msgContext.getResponseMessage().getSOAPPartAsString());
+        if (methodThread != null)
+        {
+          methodThread.abort();
+          methodThread.finishUp();
         }
       }
-      */
 
     } catch (AxisFault af) {
       log.debug(af);
@@ -401,20 +343,20 @@ public class CommonsHTTPSender extends B
     return httpChunkStream;
   }
 
-  private static String getHeader(HttpResponse response, String headerName) {
-    Header header = response.getFirstHeader(headerName);
-    return (header == null) ? null : header.getValue().trim();
+  private static String getHeader(BackgroundHTTPThread methodThread, String headerName)
+    throws IOException, InterruptedException, HttpException {
+    String header = methodThread.getFirstHeader(headerName);
+    return (header == null) ? null : header.trim();
   }
 
-  private static String getResponseBodyAsString(HttpResponse httpResponse)
-    throws IOException {
-    HttpEntity entity = httpResponse.getEntity();
-    if (entity != null)
+  private static String getResponseBodyAsString(BackgroundHTTPThread methodThread)
+    throws IOException, InterruptedException, HttpException {
+    InputStream is = methodThread.getSafeInputStream();
+    if (is != null)
     {
-      InputStream is = entity.getContent();
       try
       {
-        String charSet = EntityUtils.getContentCharSet(entity);
+        String charSet = methodThread.getCharSet();
         if (charSet == null)
           charSet = "utf-8";
         char[] buffer = new char[65536];
@@ -444,64 +386,109 @@ public class CommonsHTTPSender extends B
     return "";
   }
   
-  private static class FileBackedInputStream extends InputStream {
+  private static class MessageRequestEntity implements HttpEntity {
+
+    private final Message message;
+    private final boolean httpChunkStream; //Use HTTP chunking or not.
+    private final boolean contentLengthNeeded;
+
+    public MessageRequestEntity(Message message, boolean httpChunkStream, boolean contentLengthNeeded)
{
+      this.message = message;
+      this.httpChunkStream = httpChunkStream;
+      this.contentLengthNeeded = contentLengthNeeded;
+    }
+
+    @Override
+    public boolean isChunked() {
+      return httpChunkStream;
+    }
     
-    private InputStream fileInputStream = null;
-    private File file = null;
+    @Override
+    public void consumeContent()
+      throws IOException {
+      EntityUtils.consume(this);
+    }
     
-    public FileBackedInputStream(InputStream is)
-      throws IOException
-    {
-      File readyToOpenFile = null;
-      // Create a file and read into it
-      File tempFile = File.createTempFile("__shp__",".tmp");
-      try
-      {
-        // Open the output stream
-        OutputStream os = new FileOutputStream(tempFile);
-        try
-        {
-          byte[] buffer = new byte[65536];
-          while (true)
-          {
-            int amt = is.read(buffer);
-            if (amt == -1)
-              break;
-            os.write(buffer,0,amt);
-          }
-        }
-        finally
-        {
-          os.close();
-        }
-        readyToOpenFile = tempFile;
-        tempFile = null;
-      }
-      finally
-      {
-        if (tempFile != null)
-          tempFile.delete();
-      }
-      
-      try
-      {
-        fileInputStream = new FileInputStream(readyToOpenFile);
-        file = readyToOpenFile;
-        readyToOpenFile = null;
+    @Override
+    public boolean isRepeatable() {
+      return true;
+    }
+
+    @Override
+    public boolean isStreaming() {
+      return false;
+    }
+    
+    @Override
+    public InputStream getContent()
+      throws IOException, IllegalStateException {
+      // MHL
+      return null;
+    }
+    
+    @Override
+    public void writeTo(OutputStream out)
+      throws IOException {
+      try {
+        this.message.writeTo(out);
+      } catch (SOAPException e) {
+        throw new IOException(e.getMessage());
       }
-      finally
-      {
-        if (readyToOpenFile != null)
-          readyToOpenFile.delete();
+    }
+
+    @Override
+    public long getContentLength() {
+      if (contentLengthNeeded) {
+        try {
+          return message.getContentLength();
+        } catch (Exception e) {
+        }
       }
+      // Unknown (chunked) length
+      return -1L;
+    }
+
+    @Override
+    public Header getContentType() {
+      return null; // a separate header is added
+    }
+
+    @Override
+    public Header getContentEncoding() {
+      return null;
+    }
+  }
+
+  /** This input stream wraps a background http transaction thread, so that
+  * the thread is ended when the stream is closed.
+  */
+  private static class BackgroundInputStream extends InputStream {
+    
+    private BackgroundHTTPThread methodThread = null;
+    private InputStream xThreadInputStream = null;
+    
+    /** Construct an http transaction stream.  The stream is driven by a background
+    * thread, whose existence is tied to this class.  The sequence of activity that
+    * this class expects is as follows:
+    * (1) Construct the httpclient and request object and initialize them
+    * (2) Construct a background method thread, and start it
+    * (3) If the response calls for it, call this constructor, and put the resulting stream
+    *    into the message response
+    * (4) Otherwise, terminate the background method thread in the standard manner,
+    *    being sure NOT
+    */
+    public BackgroundInputStream(BackgroundHTTPThread methodThread, InputStream xThreadInputStream)
+    {
+      this.methodThread = methodThread;
+      this.xThreadInputStream = xThreadInputStream;
     }
     
     @Override
     public int available()
       throws IOException
     {
-      if (fileInputStream != null)
-        return fileInputStream.available();
+      if (xThreadInputStream != null)
+        return xThreadInputStream.available();
       return super.available();
     }
     
@@ -509,29 +496,37 @@ public class CommonsHTTPSender extends B
     public void close()
       throws IOException
     {
-      IOException exception = null;
       try
       {
-        if (fileInputStream != null)
-          fileInputStream.close();
+        if (xThreadInputStream != null)
+        {
+          xThreadInputStream.close();
+          xThreadInputStream = null;
+        }
       }
-      catch (IOException e)
+      finally
       {
-        exception = e;
+        if (methodThread != null)
+        {
+          methodThread.abort();
+          try
+          {
+            methodThread.finishUp();
+          }
+          catch (InterruptedException e)
+          {
+            throw new InterruptedIOException(e.getMessage());
+          }
+          methodThread = null;
+        }
       }
-      fileInputStream = null;
-      if (file != null)
-        file.delete();
-      file = null;
-      if (exception != null)
-        throw exception;
     }
     
     @Override
     public void mark(int readlimit)
     {
-      if (fileInputStream != null)
-        fileInputStream.mark(readlimit);
+      if (xThreadInputStream != null)
+        xThreadInputStream.mark(readlimit);
       else
         super.mark(readlimit);
     }
@@ -540,8 +535,8 @@ public class CommonsHTTPSender extends B
     public void reset()
       throws IOException
     {
-      if (fileInputStream != null)
-        fileInputStream.reset();
+      if (xThreadInputStream != null)
+        xThreadInputStream.reset();
       else
         super.reset();
     }
@@ -549,8 +544,8 @@ public class CommonsHTTPSender extends B
     @Override
     public boolean markSupported()
     {
-      if (fileInputStream != null)
-        return fileInputStream.markSupported();
+      if (xThreadInputStream != null)
+        return xThreadInputStream.markSupported();
       return super.markSupported();
     }
     
@@ -558,8 +553,8 @@ public class CommonsHTTPSender extends B
     public long skip(long n)
       throws IOException
     {
-      if (fileInputStream != null)
-        return fileInputStream.skip(n);
+      if (xThreadInputStream != null)
+        return xThreadInputStream.skip(n);
       return super.skip(n);
     }
     
@@ -567,8 +562,8 @@ public class CommonsHTTPSender extends B
     public int read(byte[] b, int off, int len)
       throws IOException
     {
-      if (fileInputStream != null)
-        return fileInputStream.read(b,off,len);
+      if (xThreadInputStream != null)
+        return xThreadInputStream.read(b,off,len);
       return super.read(b,off,len);
     }
 
@@ -576,8 +571,8 @@ public class CommonsHTTPSender extends B
     public int read(byte[] b)
       throws IOException
     {
-      if (fileInputStream != null)
-        return fileInputStream.read(b);
+      if (xThreadInputStream != null)
+        return xThreadInputStream.read(b);
       return super.read(b);
     }
     
@@ -585,84 +580,327 @@ public class CommonsHTTPSender extends B
     public int read()
       throws IOException
     {
-      if (fileInputStream != null)
-        return fileInputStream.read();
+      if (xThreadInputStream != null)
+        return xThreadInputStream.read();
       return -1;
     }
     
   }
-  
-  private static class MessageRequestEntity implements HttpEntity {
 
-    private final Message message;
-    private final boolean httpChunkStream; //Use HTTP chunking or not.
-    private final boolean contentLengthNeeded;
+  /** This thread does the actual socket communication with the server.
+  * It's set up so that it can be abandoned at shutdown time.
+  *
+  * The way it works is as follows:
+  * - it starts the transaction
+  * - it receives the response, and saves that for the calling class to inspect
+  * - it transfers the data part to an input stream provided to the calling class
+  * - it shuts the connection down
+  *
+  * If there is an error, the sequence is aborted, and an exception is recorded
+  * for the calling class to examine.
+  *
+  * The calling class basically accepts the sequence above.  It starts the
+  * thread, and tries to get a response code.  If instead an exception is seen,
+  * the exception is thrown up the stack.
+  */
+  protected static class BackgroundHTTPThread extends Thread
+  {
+    /** Client and method, all preconfigured */
+    protected final HttpClient httpClient;
+    protected final HttpRequestBase executeMethod;
+    
+    protected HttpResponse response = null;
+    protected Throwable responseException = null;
+    protected XThreadInputStream threadStream = null;
+    protected String charSet = null;
+    protected boolean streamCreated = false;
+    protected Throwable streamException = null;
+    protected boolean abortThread = false;
 
-    public MessageRequestEntity(Message message, boolean httpChunkStream, boolean contentLengthNeeded)
{
-      this.message = message;
-      this.httpChunkStream = httpChunkStream;
-      this.contentLengthNeeded = contentLengthNeeded;
+    protected Throwable shutdownException = null;
+
+    protected Throwable generalException = null;
+    
+    public BackgroundHTTPThread(HttpClient httpClient, HttpRequestBase executeMethod)
+    {
+      super();
+      setDaemon(true);
+      this.httpClient = httpClient;
+      this.executeMethod = executeMethod;
     }
 
-    @Override
-    public boolean isChunked() {
-      return httpChunkStream;
+    public void run()
+    {
+      try
+      {
+        try
+        {
+          // Call the execute method appropriately
+          synchronized (this)
+          {
+            if (!abortThread)
+            {
+              try
+              {
+                response = httpClient.execute(executeMethod);
+              }
+              catch (java.net.SocketTimeoutException e)
+              {
+                responseException = e;
+              }
+              catch (ConnectTimeoutException e)
+              {
+                responseException = e;
+              }
+              catch (InterruptedIOException e)
+              {
+                throw e;
+              }
+              catch (Throwable e)
+              {
+                responseException = e;
+              }
+              this.notifyAll();
+            }
+          }
+          
+          // Start the transfer of the content
+          if (responseException == null)
+          {
+            synchronized (this)
+            {
+              if (!abortThread)
+              {
+                try
+                {
+                  HttpEntity entity = response.getEntity();
+                  InputStream bodyStream = entity.getContent();
+                  if (bodyStream != null)
+                  {
+                    threadStream = new XThreadInputStream(bodyStream);
+                    charSet = EntityUtils.getContentCharSet(entity);
+                  }
+                  streamCreated = true;
+                }
+                catch (java.net.SocketTimeoutException e)
+                {
+                  streamException = e;
+                }
+                catch (ConnectTimeoutException e)
+                {
+                  streamException = e;
+                }
+                catch (InterruptedIOException e)
+                {
+                  throw e;
+                }
+                catch (Throwable e)
+                {
+                  streamException = e;
+                }
+                this.notifyAll();
+              }
+            }
+          }
+          
+          if (responseException == null && streamException == null)
+          {
+            if (threadStream != null)
+            {
+              // Stuff the content until we are done
+              threadStream.stuffQueue();
+            }
+          }
+          
+        }
+        finally
+        {
+          synchronized (this)
+          {
+            try
+            {
+              executeMethod.abort();
+            }
+            catch (Throwable e)
+            {
+              shutdownException = e;
+            }
+            this.notifyAll();
+          }
+        }
+      }
+      catch (Throwable e)
+      {
+        // We catch exceptions here that should ONLY be InterruptedExceptions, as a result
of the thread being aborted.
+        this.generalException = e;
+      }
     }
-    
-    @Override
-    public void consumeContent()
-      throws IOException {
-      EntityUtils.consume(this);
+
+    public int getResponseCode()
+      throws InterruptedException, IOException, HttpException
+    {
+      // Must wait until the response object is there
+      while (true)
+      {
+        synchronized (this)
+        {
+          checkException(responseException);
+          if (response != null)
+            return response.getStatusLine().getStatusCode();
+          wait();
+        }
+      }
     }
-    
-    @Override
-    public boolean isRepeatable() {
-      return true;
+
+    public String getResponseStatus()
+      throws InterruptedException, IOException, HttpException
+    {
+      // Must wait until the response object is there
+      while (true)
+      {
+        synchronized (this)
+        {
+          checkException(responseException);
+          if (response != null)
+            return response.getStatusLine().toString();
+          wait();
+        }
+      }
     }
 
-    @Override
-    public boolean isStreaming() {
-      return false;
+    public Map<String,List<String>> getResponseHeaders()
+      throws InterruptedException, IOException, HttpException
+    {
+      // Must wait for the response object to appear
+      while (true)
+      {
+        synchronized (this)
+        {
+          checkException(responseException);
+          if (response != null)
+          {
+            Header[] headers = response.getAllHeaders();
+            Map<String,List<String>> rval = new HashMap<String,List<String>>();
+            int i = 0;
+            while (i < headers.length)
+            {
+              Header h = headers[i++];
+              String name = h.getName();
+              String value = h.getValue();
+              List<String> values = rval.get(name);
+              if (values == null)
+              {
+                values = new ArrayList<String>();
+                rval.put(name,values);
+              }
+              values.add(value);
+            }
+            return rval;
+          }
+          wait();
+        }
+      }
+
     }
     
-    @Override
-    public InputStream getContent()
-      throws IOException, IllegalStateException {
-      // MHL
-      return null;
+    public String getFirstHeader(String headerName)
+      throws InterruptedException, IOException, HttpException
+    {
+      // Must wait for the response object to appear
+      while (true)
+      {
+        synchronized (this)
+        {
+          checkException(responseException);
+          if (response != null)
+          {
+            Header h = response.getFirstHeader(headerName);
+            if (h == null)
+              return null;
+            return h.getValue();
+          }
+          wait();
+        }
+      }
+    }
+
+    public InputStream getSafeInputStream()
+      throws InterruptedException, IOException, HttpException
+    {
+      // Must wait until stream is created, or until we note an exception was thrown.
+      while (true)
+      {
+        synchronized (this)
+        {
+          if (responseException != null)
+            throw new IllegalStateException("Check for response before getting stream");
+          checkException(streamException);
+          if (streamCreated)
+            return threadStream;
+          wait();
+        }
+      }
     }
     
-    @Override
-    public void writeTo(OutputStream out)
-      throws IOException {
-      try {
-        this.message.writeTo(out);
-      } catch (SOAPException e) {
-        throw new IOException(e.getMessage());
+    public String getCharSet()
+      throws InterruptedException, IOException, HttpException
+    {
+      while (true)
+      {
+        synchronized (this)
+        {
+          if (responseException != null)
+            throw new IllegalStateException("Check for response before getting charset");
+          checkException(streamException);
+          if (streamCreated)
+            return charSet;
+          wait();
+        }
       }
     }
-
-    @Override
-    public long getContentLength() {
-      if (contentLengthNeeded) {
-        try {
-          return message.getContentLength();
-        } catch (Exception e) {
+    
+    public void abort()
+    {
+      // This will be called during the finally
+      // block in the case where all is well (and
+      // the stream completed) and in the case where
+      // there were exceptions.
+      synchronized (this)
+      {
+        if (streamCreated)
+        {
+          if (threadStream != null)
+            threadStream.abort();
         }
+        abortThread = true;
       }
-      // Unknown (chunked) length
-      return -1L;
     }
-
-    @Override
-    public Header getContentType() {
-      return null; // a separate header is added
+    
+    public void finishUp()
+      throws InterruptedException
+    {
+      join();
     }
-
-    @Override
-    public Header getContentEncoding() {
-      return null;
+    
+    protected synchronized void checkException(Throwable exception)
+      throws IOException, HttpException
+    {
+      if (exception != null)
+      {
+        Throwable e = exception;
+        if (e instanceof IOException)
+          throw (IOException)e;
+        else if (e instanceof HttpException)
+          throw (HttpException)e;
+        else if (e instanceof RuntimeException)
+          throw (RuntimeException)e;
+        else if (e instanceof Error)
+          throw (Error)e;
+        else
+          throw new RuntimeException("Unhandled exception of type: "+e.getClass().getName(),e);
+      }
     }
+
   }
 
 }

Modified: manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CookieManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CookieManager.java?rev=1417060&r1=1417059&r2=1417060&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CookieManager.java
(original)
+++ manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CookieManager.java
Tue Dec  4 17:47:28 2012
@@ -27,8 +27,9 @@ import org.apache.manifoldcf.crawler.int
 import org.apache.manifoldcf.crawler.system.ManifoldCF;
 import org.apache.manifoldcf.crawler.system.Logging;
 
-import org.apache.commons.httpclient.Cookie;
-import org.apache.commons.httpclient.cookie.Cookie2;
+import org.apache.http.cookie.Cookie;
+import org.apache.http.cookie.ClientCookie;
+import org.apache.http.impl.cookie.BasicClientCookie2;
 
 /** This class manages the database table into which we write cookies.  The data resides
in the database,
 * as well as in cache (up to a certain point).  The result is that there is a memory limited,
database-backed repository
@@ -217,18 +218,13 @@ public class CookieManager extends org.a
         while (i < cookies.getCookieCount())
         {
           Cookie c = cookies.getCookie(i);
-          Cookie2 c2;
-          if (c instanceof Cookie2)
-            c2 = (Cookie2)c;
-          else
-            c2 = null;
           HashMap map = new HashMap();
           map.put(keyField,sessionKey);
           map.put(ordinalField,new Long(i));
           String domain = c.getDomain();
           if (domain != null && domain.length() > 0)
             map.put(domainField,domain);
-          map.put(domainSpecifiedField,booleanToString(c.isDomainAttributeSpecified()));
+          map.put(domainSpecifiedField,booleanToString(domain != null && domain.length()
> 0));
           String name = c.getName();
           if (name != null && name.length() > 0)
             map.put(nameField,name);
@@ -238,45 +234,27 @@ public class CookieManager extends org.a
           String path = c.getPath();
           if (path != null && path.length() > 0)
             map.put(pathField,path);
-          map.put(pathSpecifiedField,booleanToString(c.isPathAttributeSpecified()));
+          map.put(pathSpecifiedField,booleanToString(path != null && path.length()
> 0));
           map.put(versionField,new Long(c.getVersion()));
-          if (c2 != null)
-            map.put(versionSpecifiedField,booleanToString(c2.isVersionAttributeSpecified()));
-          else
-            // Make something up.  It may not be correct, but there's really no choice.
-            map.put(versionSpecifiedField,booleanToString(true));
+          // Make something up.  It may not be correct, but there's really no choice.
+          map.put(versionSpecifiedField,booleanToString(true));
           String comment = c.getComment();
           if (comment != null && comment.length() > 0)
             map.put(commentField,comment);
-          map.put(secureField,booleanToString(c.getSecure()));
+          map.put(secureField,booleanToString(c.isSecure()));
           Date expirationDate = c.getExpiryDate();
           if (expirationDate != null)
             map.put(expirationDateField,new Long(expirationDate.getTime()));
-          if (c2 != null)
-            map.put(discardField,booleanToString(!c2.isPersistent()));
-          else
-            // Once again, make something up.
-            map.put(discardField,booleanToString(true));
-          if (c2 != null)
-          {
-            String commentURL = c2.getCommentURL();
-            if (commentURL != null && commentURL.length() > 0)
-              map.put(commentURLField,commentURL);
-          }
-          if (c2 != null)
-          {
-            int[] ports = c2.getPorts();
-            if (ports != null && ports.length > 0)
-              map.put(portField,portsToString(ports));
-          }
-          if (c2 != null)
-            map.put(portBlankField,booleanToString(c2.isPortAttributeBlank()));
-          else
-            map.put(portBlankField,booleanToString(true));
-          if (c2 != null)
-            map.put(portSpecifiedField,booleanToString(c2.isPortAttributeSpecified()));
-          else
-            map.put(portSpecifiedField,booleanToString(false));
+          //map.put(discardField,booleanToString(!c.isPersistent()));
+          map.put(discardField,booleanToString(false));
+          String commentURL = c.getCommentURL();
+          if (commentURL != null && commentURL.length() > 0)
+            map.put(commentURLField,commentURL);
+          int[] ports = c.getPorts();
+          if (ports != null && ports.length > 0)
+            map.put(portField,portsToString(ports));
+          map.put(portBlankField,booleanToString(ports == null || ports.length == 0));
+          map.put(portSpecifiedField,booleanToString(ports != null && ports.length
> 0));
           performInsert(map,null);
           i++;
         }
@@ -330,25 +308,21 @@ public class CookieManager extends org.a
     while (i < result.getRowCount())
     {
       IResultRow row = result.getRow(i++);
-      Cookie2 c = new Cookie2();
+      String name = (String)row.getValue(nameField);
+      String value = (String)row.getValue(valueField);
+      BasicClientCookie2 c = new BasicClientCookie2(name,value);
       String domain = (String)row.getValue(domainField);
       if (domain != null && domain.length() > 0)
         c.setDomain(domain);
-      c.setDomainAttributeSpecified(stringToBoolean((String)row.getValue(domainSpecifiedField)));
-      String name = (String)row.getValue(nameField);
-      if (name != null && name.length() > 0)
-        c.setName(name);
-      String value = (String)row.getValue(valueField);
-      if (value != null && value.length() > 0)
-        c.setValue(value);
+      //c.setDomainAttributeSpecified(stringToBoolean((String)row.getValue(domainSpecifiedField)));
       String path = (String)row.getValue(pathField);
       if (path != null && path.length() > 0)
         c.setPath(path);
-      c.setPathAttributeSpecified(stringToBoolean((String)row.getValue(pathSpecifiedField)));
+      //c.setPathAttributeSpecified(stringToBoolean((String)row.getValue(pathSpecifiedField)));
       Long version = (Long)row.getValue(versionField);
       if (version != null)
         c.setVersion((int)version.longValue());
-      c.setVersionAttributeSpecified(stringToBoolean((String)row.getValue(versionSpecifiedField)));
+      //c.setVersionAttributeSpecified(stringToBoolean((String)row.getValue(versionSpecifiedField)));
       String comment = (String)row.getValue(commentField);
       if (comment != null)
         c.setComment(comment);
@@ -364,8 +338,8 @@ public class CookieManager extends org.a
       // Ports are comma-separated
       if (ports != null && ports.length() > 0)
         c.setPorts(stringToPorts(ports));
-      c.setPortAttributeBlank(stringToBoolean((String)row.getValue(portBlankField)));
-      c.setPortAttributeSpecified(stringToBoolean((String)row.getValue(portSpecifiedField)));
+      //c.setPortAttributeBlank(stringToBoolean((String)row.getValue(portBlankField)));
+      //c.setPortAttributeSpecified(stringToBoolean((String)row.getValue(portSpecifiedField)));
 
       dcs.addCookie(c);
     }
@@ -438,7 +412,7 @@ public class CookieManager extends org.a
   */
   protected static class DynamicCookieSet implements LoginCookies
   {
-    protected ArrayList cookies = new ArrayList();
+    protected List<Cookie> cookies = new ArrayList<Cookie>();
 
     public DynamicCookieSet()
     {
@@ -456,7 +430,7 @@ public class CookieManager extends org.a
 
     public Cookie getCookie(int index)
     {
-      return (Cookie)cookies.get(index);
+      return cookies.get(index);
     }
   }
 

Modified: manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CookieSet.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CookieSet.java?rev=1417060&r1=1417059&r2=1417060&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CookieSet.java
(original)
+++ manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CookieSet.java
Tue Dec  4 17:47:28 2012
@@ -18,25 +18,27 @@
 */
 package org.apache.manifoldcf.crawler.connectors.webcrawler;
 
-import org.apache.commons.httpclient.*;
+import java.util.List;
+
+import org.apache.http.cookie.Cookie;
 
 /** This class represents a bunch of cookies */
 public class CookieSet implements LoginCookies
 {
-  protected Cookie[] cookieList;
+  protected List<Cookie> cookieList;
 
-  public CookieSet(Cookie[] cookieList)
+  public CookieSet(List<Cookie> cookieList)
   {
     this.cookieList = cookieList;
   }
 
   public int getCookieCount()
   {
-    return cookieList.length;
+    return cookieList.size();
   }
 
   public Cookie getCookie(int index)
   {
-    return cookieList[index];
+    return cookieList.get(index);
   }
 }

Modified: manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CredentialsDescription.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CredentialsDescription.java?rev=1417060&r1=1417059&r2=1417060&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CredentialsDescription.java
(original)
+++ manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/CredentialsDescription.java
Tue Dec  4 17:47:28 2012
@@ -23,9 +23,9 @@ import org.apache.manifoldcf.crawler.sys
 import java.util.*;
 import java.util.regex.*;
 
-import org.apache.commons.httpclient.*;
-import org.apache.commons.httpclient.methods.*;
-import org.apache.commons.httpclient.params.*;
+import org.apache.http.auth.Credentials;
+import org.apache.http.auth.NTCredentials;
+import org.apache.http.auth.UsernamePasswordCredentials;
 
 /** This class describes credential information pulled from a configuration.
 * The data contained is organized by regular expression performed on a url.  What we store

Modified: manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/DataCache.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/DataCache.java?rev=1417060&r1=1417059&r2=1417060&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/DataCache.java
(original)
+++ manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/DataCache.java
Tue Dec  4 17:47:28 2012
@@ -26,6 +26,12 @@ import org.apache.manifoldcf.crawler.sys
 import java.util.*;
 import java.io.*;
 
+import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.http.client.RedirectException;
+import org.apache.http.client.CircularRedirectException;
+import org.apache.http.NoHttpResponseException;
+import org.apache.http.HttpException;
+
 /** This class is a cache of a specific URL's data.  It's fetched early and kept,
 * so that (1) an accurate data length can be found, and (2) we can compute a version
 * checksum.
@@ -101,7 +107,7 @@ public class DataCache
                 throw new ServiceInterruption("Socket timeout: "+e.getMessage(),e,currentTime
+ 300000L,
                   currentTime + 12 * 60 * 60000L,-1,false);
               }
-              catch (org.apache.commons.httpclient.ConnectTimeoutException e)
+              catch (ConnectTimeoutException e)
               {
                 Logging.connectors.warn("Socket connect timeout exception reading socket
stream: "+e.getMessage(),e);
                 long currentTime = System.currentTimeMillis();
@@ -179,7 +185,7 @@ public class DataCache
         {
           Logging.connectors.warn("WEB: Socket timeout exception closing data stream, ignoring:
"+e.getMessage(),e);
         }
-        catch (org.apache.commons.httpclient.ConnectTimeoutException e)
+        catch (ConnectTimeoutException e)
         {
           Logging.connectors.warn("WEB: Socket connect timeout exception closing data stream,
ignoring: "+e.getMessage(),e);
         }
@@ -199,7 +205,7 @@ public class DataCache
     {
       throw new ManifoldCFException("Socket timeout exception creating temporary file: "+e.getMessage(),e);
     }
-    catch (org.apache.commons.httpclient.ConnectTimeoutException e)
+    catch (ConnectTimeoutException e)
     {
       throw new ManifoldCFException("Socket connect timeout exception creating temporary
file: "+e.getMessage(),e);
     }

Modified: manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/LoginCookies.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/LoginCookies.java?rev=1417060&r1=1417059&r2=1417060&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/LoginCookies.java
(original)
+++ manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/LoginCookies.java
Tue Dec  4 17:47:28 2012
@@ -18,7 +18,7 @@
 */
 package org.apache.manifoldcf.crawler.connectors.webcrawler;
 
-import org.apache.commons.httpclient.*;
+import org.apache.http.cookie.Cookie;
 
 /** This interface describes cookies obtained during sequential authentication.
 */

Modified: manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/PageCredentials.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/PageCredentials.java?rev=1417060&r1=1417059&r2=1417060&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/PageCredentials.java
(original)
+++ manifoldcf/trunk/connectors/webcrawler/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/webcrawler/PageCredentials.java
Tue Dec  4 17:47:28 2012
@@ -20,9 +20,7 @@ package org.apache.manifoldcf.crawler.co
 
 import org.apache.manifoldcf.core.interfaces.*;
 
-import org.apache.commons.httpclient.*;
-import org.apache.commons.httpclient.methods.*;
-import org.apache.commons.httpclient.params.*;
+import org.apache.http.auth.Credentials;
 
 /** This interface describes immutable classes which represents authentication information
for page-based authentication.
 */



Mime
View raw message