manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1440583 - in /manifoldcf/trunk: ./ connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/
Date Wed, 30 Jan 2013 18:51:11 GMT
Author: kwright
Date: Wed Jan 30 18:51:11 2013
New Revision: 1440583

URL: http://svn.apache.org/viewvc?rev=1440583&view=rev
Log:
Fix for CONNECTORS-629.

Modified:
    manifoldcf/trunk/CHANGES.txt
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java

Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1440583&r1=1440582&r2=1440583&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Wed Jan 30 18:51:11 2013
@@ -3,6 +3,9 @@ $Id$
 
 ======================= 1.2-dev =====================
 
+CONNECTORS-629: Revamp error handling in ElasticSearch connector.
+(Andrew Clegg, Karl Wright)
+
 CONNECTORS-625: Finish internationalization work for parts of the
 core crawler UI.
 (Karl Wright)

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java?rev=1440583&r1=1440582&r2=1440583&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
Wed Jan 30 18:51:11 2013
@@ -23,6 +23,7 @@ import org.apache.http.client.methods.Ht
 import org.apache.http.client.HttpClient;
 
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
 
 public class ElasticSearchAction extends ElasticSearchConnection
 {
@@ -32,10 +33,15 @@ public class ElasticSearchAction extends
     _optimize, _refresh, _status;
   }
 
-  public ElasticSearchAction(HttpClient client, CommandEnum cmd, ElasticSearchConfig config,
boolean checkConnection)
+  public ElasticSearchAction(HttpClient client, ElasticSearchConfig config)
       throws ManifoldCFException
   {
     super(config, client);
+  }
+  
+  public void execute(CommandEnum cmd, boolean checkConnection)
+      throws ManifoldCFException, ServiceInterruption
+  {
     StringBuffer url = getApiUrl(cmd.toString(), checkConnection);
     HttpGet method = new HttpGet(url.toString());
     call(method);

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java?rev=1440583&r1=1440582&r2=1440583&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
Wed Jan 30 18:51:11 2013
@@ -56,9 +56,12 @@ import org.apache.http.HttpException;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
 
 public class ElasticSearchConnection
 {
+  protected ElasticSearchConfig config;
+  
   private HttpClient client;
   
   private String serverLocation;
@@ -83,6 +86,7 @@ public class ElasticSearchConnection
 
   protected ElasticSearchConnection(ElasticSearchConfig config, HttpClient client)
   {
+    this.config = config;
     this.client = client;
     result = Result.UNKNOWN;
     response = null;
@@ -176,7 +180,11 @@ public class ElasticSearchConnection
     }
   }
   
-  protected void call(HttpRequestBase method) throws ManifoldCFException
+  /** Call ElasticSearch.
+  *@return false if there was a "rejection".
+  */
+  protected boolean call(HttpRequestBase method)
+    throws ManifoldCFException, ServiceInterruption
   {
     CallThread ct = new CallThread(client, method);
     try
@@ -200,9 +208,8 @@ public class ElasticSearchConnection
             throw new RuntimeException("Unexpected exception thrown: "+t.getMessage(),t);
         }
         
-        if (!checkResultCode(ct.getResultCode()))
-          throw new ManifoldCFException(getResultDescription());
         response = ct.getResponse();
+        return handleResultCode(ct.getResultCode(), response);
       }
       catch (InterruptedException e)
       {
@@ -212,16 +219,67 @@ public class ElasticSearchConnection
     }
     catch (HttpException e)
     {
-      setResult(Result.ERROR, e.getMessage());
-      throw new ManifoldCFException(e);
+      handleHttpException(e);
+      return false;
     }
     catch (IOException e)
     {
-      setResult(Result.ERROR, e.getMessage());
-      throw new ManifoldCFException(e);
+      handleIOException(e);
+      return false;
     }
   }
 
+  private boolean handleResultCode(int code, String response)
+    throws ManifoldCFException, ServiceInterruption
+  {
+    if (code == 200)
+    {
+      setResult(Result.OK, null);
+      return true;
+    }
+    else if (code == 404)
+    {
+      setResult(Result.ERROR, response);
+      throw new ManifoldCFException("Server/page not found");
+    }
+    else if (code >= 400 && code < 500)
+    {
+      setResult(Result.ERROR, response);
+      return false;
+    }
+    else if (code >= 500 && code < 600)
+    {
+      setResult(Result.ERROR, "Server exception: "+response);
+      long currentTime = System.currentTimeMillis();
+      throw new ServiceInterruption("Server exception: "+response,
+        new ManifoldCFException(response),
+        currentTime + 300000L,
+        currentTime + 20L * 60000L,
+        -1,
+        false);
+    }
+    setResult(Result.UNKNOWN, response);
+    throw new ManifoldCFException("Unexpected HTTP result code: "+code+": "+response);
+  }
+
+  private void handleHttpException(HttpException e)
+    throws ManifoldCFException, ServiceInterruption {
+    setResult(Result.ERROR, e.getMessage());
+    throw new ManifoldCFException(e);
+  }
+  
+  private void handleIOException(IOException e)
+    throws ManifoldCFException, ServiceInterruption {
+    setResult(Result.ERROR, e.getMessage());
+    long currentTime = System.currentTimeMillis();
+    // All IO exceptions are treated as service interruptions, retried for an hour
+    throw new ServiceInterruption("IO exception: "+e.getMessage(),e,
+        currentTime + 60000L,
+        currentTime + 1L * 60L * 60000L,
+        -1,
+        true);
+  }
+    
   private static String getResponseBodyAsString(HttpEntity entity)
     throws IOException, HttpException {
     InputStream is = entity.getContent();
@@ -292,24 +350,6 @@ public class ElasticSearchConnection
     return response;
   }
 
-  private boolean checkResultCode(int code)
-  {
-    switch (code)
-    {
-    case 0:
-      setResult(Result.UNKNOWN, null);
-      return false;
-    case 200:
-      setResult(Result.OK, null);
-      return true;
-    case 404:
-      setResult(Result.ERROR, "Server/page not found");
-      return false;
-    default:
-      setResult(Result.ERROR, null);
-      return false;
-    }
-  }
 
   public Result getResult()
   {

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1440583&r1=1440582&r2=1440583&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
Wed Jan 30 18:51:11 2013
@@ -368,14 +368,19 @@ public class ElasticSearchConnector exte
     ElasticSearchConfig config = getConfigParameters(null);
     InputStream inputStream = document.getBinaryStream();
     long startTime = System.currentTimeMillis();
-    ElasticSearchIndex oi = new ElasticSearchIndex(client, documentURI, 
-        document, inputStream, config);
-    activities.recordActivity(startTime, ELASTICSEARCH_INDEXATION_ACTIVITY,
-      document.getBinaryLength(), documentURI, oi.getResult().name(),
-      oi.getResultDescription());
-    if (oi.getResult() != Result.OK)
-      return DOCUMENTSTATUS_REJECTED;
-    return DOCUMENTSTATUS_ACCEPTED;
+    ElasticSearchIndex oi = new ElasticSearchIndex(client, config);
+    try
+    {
+      oi.execute(documentURI, document, inputStream);
+      if (oi.getResult() != Result.OK)
+        return DOCUMENTSTATUS_REJECTED;
+      return DOCUMENTSTATUS_ACCEPTED;
+    }
+    finally
+    {
+      activities.recordActivity(startTime, ELASTICSEARCH_INDEXATION_ACTIVITY,
+        document.getBinaryLength(), documentURI, oi.getResult().name(), oi.getResultDescription());
+    }
   }
 
   @Override
@@ -385,22 +390,35 @@ public class ElasticSearchConnector exte
   {
     HttpClient client = getSession();
     long startTime = System.currentTimeMillis();
-    ElasticSearchDelete od = new ElasticSearchDelete(client, documentURI,
-        getConfigParameters(null));
-    activities.recordActivity(startTime, ELASTICSEARCH_DELETION_ACTIVITY, null,
-        documentURI, od.getResult().name(), od.getResultDescription());
+    ElasticSearchDelete od = new ElasticSearchDelete(client, getConfigParameters(null));
+    try
+    {
+      od.execute(documentURI);
+    }
+    finally
+    {
+      activities.recordActivity(startTime, ELASTICSEARCH_DELETION_ACTIVITY, null,
+          documentURI, od.getResult().name(), od.getResultDescription());
+    }
   }
 
   @Override
   public String check() throws ManifoldCFException
   {
     HttpClient client = getSession();
-    ElasticSearchAction oss = new ElasticSearchAction(client, CommandEnum._status,
-      getConfigParameters(null), true);
-    String resultName = oss.getResult().name();
-    if (resultName.equals("OK"))
-      return super.check();
-    return resultName + " " + oss.getResultDescription();
+    ElasticSearchAction oss = new ElasticSearchAction(client, getConfigParameters(null));
+    try
+    {
+      oss.execute(CommandEnum._status, true);
+      String resultName = oss.getResult().name();
+      if (resultName.equals("OK"))
+        return super.check();
+      return resultName + " " + oss.getResultDescription();
+    }
+    catch (ServiceInterruption e)
+    {
+      return "Transient exception: "+e.getMessage();
+    }
   }
 
   @Override
@@ -409,11 +427,17 @@ public class ElasticSearchConnector exte
   {
     HttpClient client = getSession();
     long startTime = System.currentTimeMillis();
-    ElasticSearchAction oo = new ElasticSearchAction(client, CommandEnum._optimize,
-        getConfigParameters(null), false);
-    activities.recordActivity(startTime, ELASTICSEARCH_OPTIMIZE_ACTIVITY, null,
-        oo.getCallUrlSnippet(), oo.getResult().name(),
-        oo.getResultDescription());
+    ElasticSearchAction oo = new ElasticSearchAction(client, getConfigParameters(null));
+    try
+    {
+      oo.execute(CommandEnum._optimize, false);
+    }
+    finally
+    {
+      activities.recordActivity(startTime, ELASTICSEARCH_OPTIMIZE_ACTIVITY, null,
+          oo.getCallUrlSnippet(), oo.getResult().name(),
+          oo.getResultDescription());
+    }
   }
 
 }

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java?rev=1440583&r1=1440582&r2=1440583&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
Wed Jan 30 18:51:11 2013
@@ -23,14 +23,19 @@ import org.apache.http.client.HttpClient
 import org.apache.http.client.methods.HttpDelete;
 
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
 
 public class ElasticSearchDelete extends ElasticSearchConnection
 {
 
-  public ElasticSearchDelete(HttpClient client, String documentURI, ElasticSearchConfig config)
-      throws ManifoldCFException
+  public ElasticSearchDelete(HttpClient client, ElasticSearchConfig config)
   {
     super(config, client);
+  }
+  
+  public void execute(String documentURI)
+      throws ManifoldCFException, ServiceInterruption
+  {
     try
     {
       String idField = java.net.URLEncoder.encode(documentURI,"utf-8");

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java?rev=1440583&r1=1440582&r2=1440583&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
Wed Jan 30 18:51:11 2013
@@ -34,6 +34,7 @@ import org.apache.http.Header;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
+import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
 import org.apache.manifoldcf.core.common.Base64;
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
 import org.apache.manifoldcf.crawler.system.Logging;
@@ -151,11 +152,17 @@ public class ElasticSearchIndex extends 
 
   }
 
-  public ElasticSearchIndex(HttpClient client, String documentURI, RepositoryDocument document,

-      InputStream inputStream, ElasticSearchConfig config) throws ManifoldCFException
+  public ElasticSearchIndex(HttpClient client, ElasticSearchConfig config)
   {
     super(config, client);
-    
+  }
+  
+  /** Do the indexing.
+  *@return false to indicate that the document was rejected.
+  */
+  public boolean execute(String documentURI, RepositoryDocument document, 
+      InputStream inputStream) throws ManifoldCFException, ServiceInterruption
+  {
     String idField;
     try
     {
@@ -169,12 +176,14 @@ public class ElasticSearchIndex extends 
     StringBuffer url = getApiUrl(config.getIndexType() + "/" + idField, false);
     HttpPut put = new HttpPut(url.toString());
     put.setEntity(new IndexRequestEntity(document, inputStream));
-    call(put);
+    if (call(put) == false)
+      return false;
     if ("true".equals(checkJson(jsonStatus)))
-      return;
+      return true;
     String error = checkJson(jsonException);
     setResult(Result.ERROR, error);
     Logging.connectors.error(getResponse());
+    return true;
   }
 
 }



Mime
View raw message