manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1438915 - in /manifoldcf/trunk: ./ connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/
Date Sat, 26 Jan 2013 16:55:14 GMT
Author: kwright
Date: Sat Jan 26 16:55:14 2013
New Revision: 1438915

URL: http://svn.apache.org/viewvc?rev=1438915&view=rev
Log:
Fix for CONNECTORS-624.

Modified:
    manifoldcf/trunk/CHANGES.txt
    manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerAction.java
    manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java
    manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
    manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerDelete.java
    manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerIndex.java
    manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerSchema.java

Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1438915&r1=1438914&r2=1438915&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Sat Jan 26 16:55:14 2013
@@ -3,6 +3,10 @@ $Id$
 
 ======================= 1.2-dev =====================
 
+CONNECTORS-624: Add indexing via subsidiary thread to OpenSearchServer
+connector.
+(Karl Wright)
+
 CONNECTORS-606: Add indexing via subsidiary thread to ElasticSearch
 connector.
 (Karl Wright)

Modified: manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerAction.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerAction.java?rev=1438915&r1=1438914&r2=1438915&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerAction.java
(original)
+++ manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerAction.java
Sat Jan 26 16:55:14 2013
@@ -17,6 +17,7 @@
 
 package org.apache.manifoldcf.agents.output.opensearchserver;
 
+import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpGet;
 
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
@@ -27,9 +28,9 @@ public class OpenSearchServerAction exte
     optimize, reload;
   }
 
-  public OpenSearchServerAction(CommandEnum cmd, OpenSearchServerConfig config)
+  public OpenSearchServerAction(HttpClient client, CommandEnum cmd, OpenSearchServerConfig
config)
       throws ManifoldCFException {
-    super(config);
+    super(client, config);
     StringBuffer url = getApiUrl("action");
     url.append("&action=");
     url.append(cmd.name());

Modified: manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java?rev=1438915&r1=1438914&r2=1438915&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java
(original)
+++ manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnection.java
Sat Jan 26 16:55:14 2013
@@ -25,6 +25,7 @@ import java.io.StringWriter;
 import java.io.Reader;
 import java.io.InputStreamReader;
 import java.io.InputStream;
+import java.io.InterruptedIOException;
 import java.net.URLEncoder;
 
 import javax.xml.parsers.DocumentBuilder;
@@ -35,26 +36,11 @@ import javax.xml.xpath.XPathExpression;
 import javax.xml.xpath.XPathExpressionException;
 import javax.xml.xpath.XPathFactory;
 
-import org.apache.http.conn.ClientConnectionManager;
-import org.apache.http.impl.conn.PoolingClientConnectionManager;
 import org.apache.http.client.HttpClient;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.impl.client.DefaultRedirectStrategy;
 import org.apache.http.client.methods.HttpRequestBase;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.methods.HttpDelete;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
-import org.apache.http.Header;
-import org.apache.http.params.CoreProtocolPNames;
-import org.apache.http.params.HttpProtocolParams;
-import org.apache.http.ProtocolVersion;
 import org.apache.http.util.EntityUtils;
-import org.apache.http.message.BasicHeader;
-import org.apache.http.client.HttpRequestRetryHandler;
-import org.apache.http.protocol.HttpContext;
 
 import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.client.RedirectException;
@@ -62,7 +48,6 @@ import org.apache.http.client.CircularRe
 import org.apache.http.NoHttpResponseException;
 import org.apache.http.HttpException;
 
-
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
 import org.w3c.dom.Document;
 import org.xml.sax.InputSource;
@@ -86,8 +71,6 @@ public class OpenSearchServerConnection 
 
   private Document xmlResponse;
 
-  private ClientConnectionManager connectionManager;
-  
   private HttpClient httpClient;
   
   protected String xPathStatus = "/response/entry[@key='Status']/text()";
@@ -99,7 +82,8 @@ public class OpenSearchServerConnection 
 
   private Result result;
 
-  protected OpenSearchServerConnection(OpenSearchServerConfig config) {
+  protected OpenSearchServerConnection(HttpClient client, OpenSearchServerConfig config)
{
+    this.httpClient = client;
     result = Result.UNKNOWN;
     response = null;
     xmlResponse = null;
@@ -109,24 +93,6 @@ public class OpenSearchServerConnection 
     indexName = config.getIndexName();
     userName = config.getUserName();
     apiKey = config.getApiKey();
-    PoolingClientConnectionManager localConnectionManager = new PoolingClientConnectionManager();
-    localConnectionManager.setMaxTotal(1);
-    connectionManager = localConnectionManager;
-    DefaultHttpClient localHttpClient = new DefaultHttpClient(connectionManager);
-    // No retries
-    localHttpClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
-      {
-	public boolean retryRequest(
-	  IOException exception,
-	  int executionCount,
-          HttpContext context)
-	{
-	  return false;
-	}
-     
-      });
-
-    httpClient = localHttpClient;
   }
 
   protected final String urlEncode(String t) throws ManifoldCFException {
@@ -155,21 +121,110 @@ public class OpenSearchServerConnection 
     return url;
   }
 
-  protected void call(HttpRequestBase method) throws ManifoldCFException {
-    try {
-      HttpResponse resp = httpClient.execute(method);
-      if (!checkResultCode(resp.getStatusLine().getStatusCode()))
-        throw new ManifoldCFException(getResultDescription());
-      response = getResponseBodyAsString(resp.getEntity());
-    } catch (HttpException e) {
+  protected static class CallThread extends Thread
+  {
+    protected final HttpClient client;
+    protected final HttpRequestBase method;
+    protected int resultCode = -1;
+    protected String response = null;
+    protected Throwable exception = null;
+    
+    public CallThread(HttpClient client, HttpRequestBase method)
+    {
+      this.client = client;
+      this.method = method;
+      setDaemon(true);
+    }
+    
+    @Override
+    public void run()
+    {
+      try
+      {
+        try
+        {
+          HttpResponse resp = client.execute(method);
+          resultCode = resp.getStatusLine().getStatusCode();
+          response = getResponseBodyAsString(resp.getEntity());
+        }
+        finally
+        {
+          method.abort();
+        }
+      }
+      catch (java.net.SocketTimeoutException e)
+      {
+        exception = e;
+      }
+      catch (InterruptedIOException e)
+      {
+        // Just exit
+      }
+      catch (Throwable e)
+      {
+        exception = e;
+      }
+    }
+    
+    public int getResultCode()
+    {
+      return resultCode;
+    }
+    
+    public String getResponse()
+    {
+      return response;
+    }
+    
+    public Throwable getException()
+    {
+      return exception;
+    }
+  }
+  
+  protected void call(HttpRequestBase method) throws ManifoldCFException
+  {
+    CallThread ct = new CallThread(httpClient, method);
+    try
+    {
+      ct.start();
+      try
+      {
+        ct.join();
+        Throwable t = ct.getException();
+        if (t != null)
+        {
+          if (t instanceof HttpException)
+            throw (HttpException)t;
+          else if (t instanceof IOException)
+            throw (IOException)t;
+          else if (t instanceof RuntimeException)
+            throw (RuntimeException)t;
+          else if (t instanceof Error)
+            throw (Error)t;
+          else
+            throw new RuntimeException("Unexpected exception thrown: "+t.getMessage(),t);
+        }
+        
+        if (!checkResultCode(ct.getResultCode()))
+          throw new ManifoldCFException(getResultDescription());
+        response = ct.getResponse();
+      }
+      catch (InterruptedException e)
+      {
+        ct.interrupt();
+        throw new ManifoldCFException("Interrupted: "+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+      }
+    }
+    catch (HttpException e)
+    {
       setResult(Result.ERROR, e.getMessage());
       throw new ManifoldCFException(e);
-    } catch (IOException e) {
+    }
+    catch (IOException e)
+    {
       setResult(Result.ERROR, e.getMessage());
       throw new ManifoldCFException(e);
-    } finally {
-      if (method != null)
-        method.abort();
     }
   }
 

Modified: manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java?rev=1438915&r1=1438914&r2=1438915&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
(original)
+++ manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerConnector.java
Sat Jan 26 16:55:14 2013
@@ -28,6 +28,13 @@ import java.util.Map;
 import java.util.TreeMap;
 import java.util.Locale;
 
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.impl.conn.PoolingClientConnectionManager;
+import org.apache.http.client.HttpClient;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.protocol.HttpContext;
+
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
@@ -86,6 +93,13 @@ public class OpenSearchServerConnector e
   /** Forward to the javascript to check the specification parameters for the job */
   private static final String EDIT_SPEC_HEADER_FORWARD = "editSpecification.js";
 
+  /** Connection expiration interval */
+  private static final long EXPIRATION_INTERVAL = 60000L;
+
+  private ClientConnectionManager connectionManager = null;
+  private HttpClient client = null;
+  private long expirationTime = -1L;
+
   // Private data
 
   private String specsCacheOutpuDescription;
@@ -97,6 +111,73 @@ public class OpenSearchServerConnector e
   }
 
   @Override
+  public void connect(ConfigParams configParams)
+  {
+    super.connect(configParams);
+  }
+  
+  protected HttpClient getSession()
+    throws ManifoldCFException
+  {
+    if (client == null)
+    {
+      PoolingClientConnectionManager localConnectionManager = new PoolingClientConnectionManager();
+      localConnectionManager.setMaxTotal(1);
+      connectionManager = localConnectionManager;
+      DefaultHttpClient localClient = new DefaultHttpClient(connectionManager);
+      // No retries
+      localClient.setHttpRequestRetryHandler(new HttpRequestRetryHandler()
+        {
+          public boolean retryRequest(
+            IOException exception,
+            int executionCount,
+            HttpContext context)
+          {
+            return false;
+          }
+       
+        });
+      client = localClient;
+    }
+    expirationTime = System.currentTimeMillis() + EXPIRATION_INTERVAL;
+    return client;
+  }
+
+  protected void closeSession()
+  {
+    if (connectionManager != null)
+    {
+      connectionManager.shutdown();
+      connectionManager = null;
+    }
+    client = null;
+    expirationTime = -1L;
+  }
+  
+  @Override
+  public void disconnect()
+    throws ManifoldCFException
+  {
+    super.disconnect();
+    closeSession();
+  }
+  
+  
+  @Override
+  public void poll()
+    throws ManifoldCFException
+  {
+    super.poll();
+    if (connectionManager != null)
+    {
+      if (System.currentTimeMillis() > expirationTime)
+      {
+        closeSession();
+      }
+    }
+  }
+
+  @Override
   public String[] getActivitiesList() {
     return OPENSEARCHSERVER_ACTIVITIES;
   }
@@ -304,13 +385,16 @@ public class OpenSearchServerConnector e
       RepositoryDocument document, String authorityNameString,
       IOutputAddActivity activities) throws ManifoldCFException,
       ServiceInterruption {
+    HttpClient client = getSession();
     OpenSearchServerConfig config = getConfigParameters(null);
     Integer count = addInstance(config);
     synchronized (count) {
       InputStream inputStream = document.getBinaryStream();
       try {
         long startTime = System.currentTimeMillis();
-        OpenSearchServerIndex oi = new OpenSearchServerIndex(documentURI,
+        OpenSearchServerIndex oi = new OpenSearchServerIndex(
+            client,
+            documentURI,
             inputStream, config);
         activities.recordActivity(startTime,
             OPENSEARCHSERVER_INDEXATION_ACTIVITY, document.getBinaryLength(),
@@ -328,8 +412,11 @@ public class OpenSearchServerConnector e
   public void removeDocument(String documentURI, String outputDescription,
       IOutputRemoveActivity activities) throws ManifoldCFException,
       ServiceInterruption {
+    HttpClient client = getSession();
     long startTime = System.currentTimeMillis();
-    OpenSearchServerDelete od = new OpenSearchServerDelete(documentURI,
+    OpenSearchServerDelete od = new OpenSearchServerDelete(
+        client,
+        documentURI,
         getConfigParameters(null));
     activities.recordActivity(startTime, OPENSEARCHSERVER_DELETION_ACTIVITY,
         null, documentURI, od.getResult().name(), od.getResultDescription());
@@ -337,7 +424,9 @@ public class OpenSearchServerConnector e
 
   @Override
   public String check() throws ManifoldCFException {
+    HttpClient client = getSession();
     OpenSearchServerSchema oss = new OpenSearchServerSchema(
+        client,
         getConfigParameters(null));
     return oss.getResult().name() + " " + oss.getResultDescription();
   }
@@ -345,8 +434,10 @@ public class OpenSearchServerConnector e
   @Override
   public void noteJobComplete(IOutputNotifyActivity activities)
       throws ManifoldCFException, ServiceInterruption {
+    HttpClient client = getSession();
     long startTime = System.currentTimeMillis();
     OpenSearchServerAction oo = new OpenSearchServerAction(
+        client,
         CommandEnum.optimize, getConfigParameters(null));
     activities.recordActivity(startTime, OPENSEARCHSERVER_OPTIMIZE_ACTIVITY,
         null, oo.getCallUrlSnippet(), oo.getResult().name(),

Modified: manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerDelete.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerDelete.java?rev=1438915&r1=1438914&r2=1438915&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerDelete.java
(original)
+++ manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerDelete.java
Sat Jan 26 16:55:14 2013
@@ -17,15 +17,16 @@
 
 package org.apache.manifoldcf.agents.output.opensearchserver;
 
+import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpGet;
 
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
 
 public class OpenSearchServerDelete extends OpenSearchServerConnection {
 
-  public OpenSearchServerDelete(String documentURI,
+  public OpenSearchServerDelete(HttpClient client, String documentURI,
       OpenSearchServerConfig config) throws ManifoldCFException {
-    super(config);
+    super(client, config);
     StringBuffer url = getApiUrl("delete");
     url.append("&uniq=");
     url.append(urlEncode(documentURI));

Modified: manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerIndex.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerIndex.java?rev=1438915&r1=1438914&r2=1438915&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerIndex.java
(original)
+++ manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerIndex.java
Sat Jan 26 16:55:14 2013
@@ -22,6 +22,7 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.PrintWriter;
 
+import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.HttpEntity;
 import org.apache.http.message.BasicHeader;
@@ -118,9 +119,9 @@ public class OpenSearchServerIndex exten
 
   }
 
-  public OpenSearchServerIndex(String documentURI, InputStream inputStream,
+  public OpenSearchServerIndex(HttpClient client, String documentURI, InputStream inputStream,
       OpenSearchServerConfig config) throws ManifoldCFException {
-    super(config);
+    super(client, config);
     StringBuffer url = getApiUrl("update");
     HttpPut put = new HttpPut(url.toString());
     put.setEntity(new IndexRequestEntity(documentURI, inputStream));

Modified: manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerSchema.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerSchema.java?rev=1438915&r1=1438914&r2=1438915&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerSchema.java
(original)
+++ manifoldcf/trunk/connectors/opensearchserver/connector/src/main/java/org/apache/manifoldcf/agents/output/opensearchserver/OpenSearchServerSchema.java
Sat Jan 26 16:55:14 2013
@@ -17,15 +17,16 @@
 
 package org.apache.manifoldcf.agents.output.opensearchserver;
 
+import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpGet;
 
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
 
 public class OpenSearchServerSchema extends OpenSearchServerConnection {
 
-  public OpenSearchServerSchema(OpenSearchServerConfig config)
+  public OpenSearchServerSchema(HttpClient client, OpenSearchServerConfig config)
       throws ManifoldCFException {
-    super(config);
+    super(client, config);
     String indexName = config.getIndexName();
     StringBuffer url = getApiUrl("schema");
     url.append("&cmd=indexList");



Mime
View raw message