incubator-connectors-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1294377 - /incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/
Date Mon, 27 Feb 2012 22:48:03 GMT
Author: kwright
Date: Mon Feb 27 22:48:02 2012
New Revision: 1294377

URL: http://svn.apache.org/viewvc?rev=1294377&view=rev
Log:
Clean up a couple of things; for one thing disable internal caching until we sort out the
problems.

Modified:
    incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
    incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
    incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
    incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
    incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java

Modified: incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java?rev=1294377&r1=1294376&r2=1294377&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
(original)
+++ incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
Mon Feb 27 22:48:02 2012
@@ -18,6 +18,7 @@
 package org.apache.manifoldcf.agents.output.elasticsearch;
 
 import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.HttpClient;
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
 
 public class ElasticSearchAction extends ElasticSearchConnection
@@ -28,13 +29,12 @@ public class ElasticSearchAction extends
     _optimize, _refresh, _status;
   }
 
-  public ElasticSearchAction(CommandEnum cmd, ElasticSearchConfig config)
+  public ElasticSearchAction(HttpClient client, CommandEnum cmd, ElasticSearchConfig config)
       throws ManifoldCFException
   {
-    super(config);
+    super(config, client);
     StringBuffer url = getApiUrl(cmd.toString());
     GetMethod method = new GetMethod(url.toString());
-    System.out.println("URL is '"+url.toString()+"'");
     call(method);
     if ("true".equals(checkJson(jsonStatus)))
       return;

Modified: incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java?rev=1294377&r1=1294376&r2=1294377&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
(original)
+++ incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
Mon Feb 27 22:48:02 2012
@@ -21,18 +21,16 @@ import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
 
-import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.httpclient.HttpException;
 import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
+import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.io.IOUtils;
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
 
 public class ElasticSearchConnection
 {
-
-  private MultiThreadedHttpConnectionManager connectionManager;
-
+  private HttpClient client;
+  
   private String serverLocation;
 
   private String indexName;
@@ -57,8 +55,9 @@ public class ElasticSearchConnection
 
   private Result result;
 
-  protected ElasticSearchConnection(ElasticSearchConfig config)
+  protected ElasticSearchConnection(ElasticSearchConfig config, HttpClient client)
   {
+    this.client = client;
     result = Result.UNKNOWN;
     response = null;
     resultDescription = "";
@@ -67,8 +66,6 @@ public class ElasticSearchConnection
     indexName = config.getIndexName();
     userName = config.getUserName();
     apiKey = config.getApiKey();
-    connectionManager = new MultiThreadedHttpConnectionManager();
-    connectionManager.getParams().setMaxTotalConnections(1);
   }
 
   protected final String urlEncode(String t) throws ManifoldCFException
@@ -103,7 +100,7 @@ public class ElasticSearchConnection
 
   protected void call(HttpMethod method) throws ManifoldCFException
   {
-    HttpClient hc = new HttpClient(connectionManager);
+    HttpClient hc = client;
     try
     {
       hc.executeMethod(method);

Modified: incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1294377&r1=1294376&r2=1294377&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
(original)
+++ incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
Mon Feb 27 22:48:02 2012
@@ -42,6 +42,8 @@ import org.apache.manifoldcf.core.interf
 import org.apache.manifoldcf.core.interfaces.IThreadContext;
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
 import org.apache.manifoldcf.core.interfaces.SpecificationNode;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
 import org.json.JSONException;
 import org.json.JSONObject;
 
@@ -53,9 +55,9 @@ import org.json.JSONObject;
 public class ElasticSearchConnector extends BaseOutputConnector
 {
 
-  private final static String ELASTICSEARCH_INDEXATION_ACTIVITY = "Optimize";
+  private final static String ELASTICSEARCH_INDEXATION_ACTIVITY = "Indexation";
   private final static String ELASTICSEARCH_DELETION_ACTIVITY = "Deletion";
-  private final static String ELASTICSEARCH_OPTIMIZE_ACTIVITY = "Indexation";
+  private final static String ELASTICSEARCH_OPTIMIZE_ACTIVITY = "Optimize";
 
   private final static String[] ELASTICSEARCH_ACTIVITIES =
   { ELASTICSEARCH_INDEXATION_ACTIVITY, ELASTICSEARCH_DELETION_ACTIVITY,
@@ -89,16 +91,44 @@ public class ElasticSearchConnector exte
   /** Forward to the javascript to check the specification parameters for the job */
   private static final String EDIT_SPEC_HEADER_FORWARD = "editSpecification.js";
 
-  private String specsCacheOutpuDescription;
-  private ElasticSearchSpecs specsCache;
+  private MultiThreadedHttpConnectionManager connectionManager = null;
+  private HttpClient client = null;
+  private String specsCacheOutpuDescription = null;
+  private ElasticSearchSpecs specsCache = null;
 
   public ElasticSearchConnector()
   {
-    specsCacheOutpuDescription = null;
-    specsCache = null;
   }
 
   @Override
+  public void connect(ConfigParams configParams)
+  {
+    super.connect(configParams);
+    connectionManager = new MultiThreadedHttpConnectionManager();
+    connectionManager.getParams().setMaxTotalConnections(1);
+    client = new HttpClient(connectionManager);
+  }
+  
+  @Override
+  public void disconnect()
+    throws ManifoldCFException
+  {
+    super.disconnect();
+    connectionManager.shutdown();
+    connectionManager = null;
+    client = null;
+  }
+  
+  @Override
+  public void poll()
+    throws ManifoldCFException
+  {
+    super.poll();
+    // Free idle connections in the pool.
+    // MHL
+  }
+  
+  @Override
   public String[] getActivitiesList()
   {
     return ELASTICSEARCH_ACTIVITIES;
@@ -213,10 +243,7 @@ public class ElasticSearchConnector exte
   {
     if (configParams == null)
       configParams = getConfiguration();
-    synchronized (this)
-    {
-      return new ElasticSearchConfig(configParams);
-    }
+    return new ElasticSearchConfig(configParams);
   }
 
   final private ElasticSearchSpecs getSpecParameters(OutputSpecification os)
@@ -230,6 +257,7 @@ public class ElasticSearchConnector exte
   {
     try
     {
+      /*
       synchronized (this)
       {
         if (!outputDescription.equals(specsCacheOutpuDescription))
@@ -238,6 +266,8 @@ public class ElasticSearchConnector exte
           specsCache = new ElasticSearchSpecs(new JSONObject(outputDescription));
         return specsCache;
       }
+      */
+      return new ElasticSearchSpecs(new JSONObject(outputDescription));
     } catch (JSONException e)
     {
       throw new ManifoldCFException(e);
@@ -305,41 +335,6 @@ public class ElasticSearchConnector exte
     return null;
   }
 
-  private static Map<String, Integer> ossInstances = null;
-
-  private synchronized final Integer addInstance(ElasticSearchConfig config)
-  {
-    if (ossInstances == null)
-      ossInstances = new TreeMap<String, Integer>();
-    synchronized (ossInstances)
-    {
-      String uii = config.getUniqueIndexIdentifier();
-      Integer count = ossInstances.get(uii);
-      if (count == null)
-      {
-        count = new Integer(1);
-        ossInstances.put(uii, count);
-      } else
-        count++;
-      return count;
-    }
-  }
-
-  private synchronized final void removeInstance(ElasticSearchConfig config)
-  {
-    if (ossInstances == null)
-      return;
-    synchronized (ossInstances)
-    {
-      String uii = config.getUniqueIndexIdentifier();
-      Integer count = ossInstances.get(uii);
-      if (count == null)
-        return;
-      if (--count == 0)
-        ossInstances.remove(uii);
-    }
-  }
-
   @Override
   public int addOrReplaceDocument(String documentURI, String outputDescription,
       RepositoryDocument document, String authorityNameString,
@@ -347,26 +342,16 @@ public class ElasticSearchConnector exte
       ServiceInterruption
   {
     ElasticSearchConfig config = getConfigParameters(null);
-    Integer count = addInstance(config);
-    synchronized (count)
-    {
-      InputStream inputStream = document.getBinaryStream();
-      try
-      {
-        long startTime = System.currentTimeMillis();
-        ElasticSearchIndex oi = new ElasticSearchIndex(documentURI,
-            inputStream, config);
-        activities.recordActivity(startTime, ELASTICSEARCH_INDEXATION_ACTIVITY,
-            document.getBinaryLength(), documentURI, oi.getResult().name(),
-            oi.getResultDescription());
-        if (oi.getResult() != Result.OK)
-          return DOCUMENTSTATUS_REJECTED;
-      } finally
-      {
-        removeInstance(config);
-      }
-      return DOCUMENTSTATUS_ACCEPTED;
-    }
+    InputStream inputStream = document.getBinaryStream();
+    long startTime = System.currentTimeMillis();
+    ElasticSearchIndex oi = new ElasticSearchIndex(client, documentURI,
+      inputStream, config);
+    activities.recordActivity(startTime, ELASTICSEARCH_INDEXATION_ACTIVITY,
+      document.getBinaryLength(), documentURI, oi.getResult().name(),
+      oi.getResultDescription());
+    if (oi.getResult() != Result.OK)
+      return DOCUMENTSTATUS_REJECTED;
+    return DOCUMENTSTATUS_ACCEPTED;
   }
 
   @Override
@@ -375,7 +360,7 @@ public class ElasticSearchConnector exte
       ServiceInterruption
   {
     long startTime = System.currentTimeMillis();
-    ElasticSearchDelete od = new ElasticSearchDelete(documentURI,
+    ElasticSearchDelete od = new ElasticSearchDelete(client, documentURI,
         getConfigParameters(null));
     activities.recordActivity(startTime, ELASTICSEARCH_DELETION_ACTIVITY, null,
         documentURI, od.getResult().name(), od.getResultDescription());
@@ -384,20 +369,12 @@ public class ElasticSearchConnector exte
   @Override
   public String check() throws ManifoldCFException
   {
-    try
-    {
-      ElasticSearchAction oss = new ElasticSearchAction(CommandEnum._status,
-          getConfigParameters(null));
-      String resultName = oss.getResult().name();
-      if (resultName.equals("OK"))
-        return super.check();
-      return resultName + " " + oss.getResultDescription();
-    }
-    catch (ManifoldCFException e)
-    {
-      e.printStackTrace();
-      throw e;
-    }
+    ElasticSearchAction oss = new ElasticSearchAction(client, CommandEnum._status,
+      getConfigParameters(null));
+    String resultName = oss.getResult().name();
+    if (resultName.equals("OK"))
+      return super.check();
+    return resultName + " " + oss.getResultDescription();
   }
 
   @Override
@@ -405,7 +382,7 @@ public class ElasticSearchConnector exte
       throws ManifoldCFException, ServiceInterruption
   {
     long startTime = System.currentTimeMillis();
-    ElasticSearchAction oo = new ElasticSearchAction(CommandEnum._optimize,
+    ElasticSearchAction oo = new ElasticSearchAction(client, CommandEnum._optimize,
         getConfigParameters(null));
     activities.recordActivity(startTime, ELASTICSEARCH_OPTIMIZE_ACTIVITY, null,
         oo.getCallUrlSnippet(), oo.getResult().name(),

Modified: incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java?rev=1294377&r1=1294376&r2=1294377&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
(original)
+++ incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
Mon Feb 27 22:48:02 2012
@@ -18,22 +18,26 @@
 package org.apache.manifoldcf.agents.output.elasticsearch;
 
 import org.apache.commons.httpclient.methods.DeleteMethod;
+import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
 
 public class ElasticSearchDelete extends ElasticSearchConnection
 {
 
-  public ElasticSearchDelete(String documentURI, ElasticSearchConfig config)
+  public ElasticSearchDelete(HttpClient client, String documentURI, ElasticSearchConfig config)
       throws ManifoldCFException
   {
-    super(config);
+    super(config, client);
     String fileName = FilenameUtils.getName(documentURI);
     DeleteMethod method = new DeleteMethod(config.getServerLocation());
     method.setPath("/" + config.getIndexName() + "/" + config.getIndexType()
         + "/" + fileName);
+    System.out.println("Deleting '"+documentURI+"'...");
     call(method);
-    if ("true".equals(checkJson(jsonStatus)))
+    System.out.println("... completed");
+    System.out.println(jsonStatus);
+    if ("ok".equals(jsonStatus))
       return;
     setResult(Result.ERROR, checkJson(jsonException));
   }

Modified: incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java?rev=1294377&r1=1294376&r2=1294377&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
(original)
+++ incubator/lcf/branches/CONNECTORS-288/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
Mon Feb 27 22:48:02 2012
@@ -23,6 +23,7 @@ import java.io.OutputStream;
 import java.io.PrintWriter;
 
 import org.apache.commons.httpclient.methods.PutMethod;
+import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.httpclient.methods.RequestEntity;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.io.IOUtils;
@@ -88,10 +89,10 @@ public class ElasticSearchIndex extends 
     }
   }
 
-  public ElasticSearchIndex(String documentURI, InputStream inputStream,
+  public ElasticSearchIndex(HttpClient client, String documentURI, InputStream inputStream,
       ElasticSearchConfig config) throws ManifoldCFException
   {
-    super(config);
+    super(config, client);
     String fileName = FilenameUtils.getName(documentURI);
     StringBuffer url = getApiUrl(config.getIndexType() + "/" + fileName);
     PutMethod put = new PutMethod(url.toString());



Mime
View raw message