manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mol...@apache.org
Subject svn commit: r1633348 - in /manifoldcf/trunk: connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/
Date Tue, 21 Oct 2014 11:43:17 GMT
Author: molgun
Date: Tue Oct 21 11:43:17 2014
New Revision: 1633348

URL: http://svn.apache.org/r1633348
Log:
CONNECTORS-1077: For ElasticSearch

Modified:
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
    manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java?rev=1633348&r1=1633347&r2=1633348&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
Tue Oct 21 11:43:17 2014
@@ -28,6 +28,7 @@ import java.io.InputStream;
 import java.io.InterruptedIOException;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
+import java.util.Locale;
 
 import org.apache.http.client.HttpClient;
 import org.apache.http.impl.client.DefaultRedirectStrategy;
@@ -51,6 +52,7 @@ import org.apache.http.NoHttpResponseExc
 import org.apache.http.HttpException;
 import org.apache.http.ParseException;
 
+import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
 import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
 import org.apache.manifoldcf.core.util.URLEncoder;
@@ -71,6 +73,8 @@ public class ElasticSearchConnection
 
   private String response;
 
+  private String resultCode;
+
   protected final static String jsonException = "\"error\"";
 
   public enum Result
@@ -225,22 +229,22 @@ public class ElasticSearchConnection
   {
     if (code == 200 || code == 201)
     {
-      setResult(Result.OK, null);
+      setResult("OK",Result.OK, null);
       return true;
     }
     else if (code == 404)
     {
-      setResult(Result.ERROR, "Page not found: " + response);
+      setResult(IOutputHistoryActivity.HTTP_ERROR,Result.ERROR, "Page not found: " + response);
       throw new ManifoldCFException("Server/page not found");
     }
     else if (code >= 400 && code < 500)
     {
-      setResult(Result.ERROR, "HTTP code = "+code+", Response = "+response);
+      setResult(IOutputHistoryActivity.HTTP_ERROR,Result.ERROR, "HTTP code = "+code+", Response
= "+response);
       return false;
     }
     else if (code >= 500 && code < 600)
     {
-      setResult(Result.ERROR, "Server exception: "+response);
+      setResult(IOutputHistoryActivity.HTTP_ERROR,Result.ERROR, "Server exception: "+response);
       long currentTime = System.currentTimeMillis();
       throw new ServiceInterruption("Server exception: "+response,
         new ManifoldCFException(response),
@@ -249,13 +253,13 @@ public class ElasticSearchConnection
         -1,
         false);
     }
-    setResult(Result.UNKNOWN, "HTTP code = "+code+", Response = "+response);
+    setResult(IOutputHistoryActivity.HTTP_ERROR,Result.UNKNOWN, "HTTP code = "+code+", Response
= "+response);
     throw new ManifoldCFException("Unexpected HTTP result code: "+code+": "+response);
   }
 
   protected void handleHttpException(HttpException e)
     throws ManifoldCFException, ServiceInterruption {
-    setResult(Result.ERROR, e.getMessage());
+    setResult(e.getClass().getSimpleName().toUpperCase(Locale.ROOT),Result.ERROR, e.getMessage());
     throw new ManifoldCFException(e);
   }
   
@@ -263,7 +267,7 @@ public class ElasticSearchConnection
     throws ManifoldCFException, ServiceInterruption {
     if (e instanceof java.io.InterruptedIOException && !(e instanceof java.net.SocketTimeoutException))
       throw new ManifoldCFException(e.getMessage(),ManifoldCFException.INTERRUPTED);
-    setResult(Result.ERROR, e.getMessage());
+    setResult(e.getClass().getSimpleName().toUpperCase(Locale.ROOT),Result.ERROR, e.getMessage());
     long currentTime = System.currentTimeMillis();
     // All IO exceptions are treated as service interruptions, retried for an hour
     throw new ServiceInterruption("IO exception: "+e.getMessage(),e,
@@ -334,13 +338,14 @@ public class ElasticSearchConnection
     return result;
   }
 
-  protected void setResult(Result res, String desc)
+  protected void setResult(String resultCode, Result res, String desc)
   {
     if (res != null)
       result = res;
     if (desc != null)
       if (desc.length() > 0)
         resultDescription = desc;
+    setResultCode(resultCode);
   }
 
   public String getResultDescription()
@@ -363,4 +368,8 @@ public class ElasticSearchConnection
   {
     return callUrlSnippet;
   }
+
+  public String getResultCode(){ return resultCode; }
+
+  public void setResultCode(String resultCode){ this.resultCode = resultCode; }
 }

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1633348&r1=1633347&r2=1633348&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
Tue Oct 21 11:43:17 2014
@@ -365,6 +365,7 @@ public class ElasticSearchConnector exte
       else
       {
         // Don't know how to deal with it
+        activities.recordActivity(null,ELASTICSEARCH_INDEXATION_ACTIVITY,document.getBinaryLength(),documentURI,activities.UNKNOWN_SECURITY,"Rejected
document that has security info which ElasticSearch does not recognize: '"+ securityType +
"'");
         return DOCUMENTSTATUS_REJECTED;
       }
     }
@@ -381,7 +382,7 @@ public class ElasticSearchConnector exte
     finally
     {
       activities.recordActivity(startTime, ELASTICSEARCH_INDEXATION_ACTIVITY,
-        document.getBinaryLength(), documentURI, oi.getResult().name(), oi.getResultDescription());
+        document.getBinaryLength(), documentURI, oi.getResultCode(), oi.getResultDescription());
     }
   }
 
@@ -400,7 +401,7 @@ public class ElasticSearchConnector exte
     finally
     {
       activities.recordActivity(startTime, ELASTICSEARCH_DELETION_ACTIVITY, null,
-          documentURI, od.getResult().name(), od.getResultDescription());
+          documentURI, od.getResultCode(), od.getResultDescription());
     }
   }
 
@@ -437,7 +438,7 @@ public class ElasticSearchConnector exte
     finally
     {
       activities.recordActivity(startTime, ELASTICSEARCH_OPTIMIZE_ACTIVITY, null,
-          oo.getCallUrlSnippet(), oo.getResult().name(),
+          oo.getCallUrlSnippet(), oo.getResultCode(),
           oo.getResultDescription());
     }
   }

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java?rev=1633348&r1=1633347&r2=1633348&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
Tue Oct 21 11:43:17 2014
@@ -22,6 +22,7 @@ package org.apache.manifoldcf.agents.out
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpDelete;
 
+import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
 import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
 import org.apache.manifoldcf.core.util.URLEncoder;
@@ -48,7 +49,7 @@ public class ElasticSearchDelete extends
         return;
       // We thought we needed to delete, but ElasticSearch disagreed.
       // Log the result as an error, but proceed anyway.
-      setResult(Result.ERROR, error);
+      setResult(IOutputHistoryActivity.JSON_ERROR,Result.ERROR, error);
       Logging.connectors.warn("ES: Delete failed: "+getResponse());
   }
 }

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java?rev=1633348&r1=1633347&r2=1633348&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
Tue Oct 21 11:43:17 2014
@@ -35,6 +35,7 @@ import org.apache.http.message.BasicHead
 import org.apache.http.Header;
 
 import org.apache.commons.io.IOUtils;
+import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
 import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
 import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
 import org.apache.manifoldcf.agents.output.elasticsearch.ElasticSearchConnection.Result;
@@ -280,7 +281,7 @@ public class ElasticSearchIndex extends 
     String error = checkJson(jsonException);
     if (getResult() == Result.OK && error == null)
       return true;
-    setResult(Result.ERROR, error);
+    setResult(IOutputHistoryActivity.JSON_ERROR,Result.ERROR, error);
     Logging.connectors.warn("ES: Index failed: "+getResponse());
     return true;
   }

Modified: manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java?rev=1633348&r1=1633347&r2=1633348&view=diff
==============================================================================
--- manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java
(original)
+++ manifoldcf/trunk/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/IOutputHistoryActivity.java
Tue Oct 21 11:43:17 2014
@@ -35,7 +35,9 @@ public interface IOutputHistoryActivity
   public static final String EXCLUDED_LENGTH = "EXCLUDEDLENGTH";
   public static final String EXCLUDED_MIMETYPE = "EXCLUDEDMIMETYPE";
   public static final String EXCLUDED_DATE = "EXCLUDEDDATE";
-  
+  public static final String UNKNOWN_SECURITY = "UNKNOWNSECURITY";
+  public static final String HTTP_ERROR = "HTTPERROR";
+  public static final String JSON_ERROR = "JSONERROR";
   /** Record time-stamped information about the activity of the output connector.
   *@param startTime is either null or the time since the start of epoch in milliseconds (Jan
1, 1970).  Every
   *       activity has an associated time; the startTime field records when the activity
began.  A null value



Mime
View raw message