manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1847863 - in /manifoldcf/trunk/connectors/elasticsearch/connector/src/main: java/org/apache/manifoldcf/agents/output/elasticsearch/ native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/ resources/org/apache/manifoldcf/agents/outp...
Date Fri, 30 Nov 2018 23:34:01 GMT
Author: kwright
Date: Fri Nov 30 23:34:01 2018
New Revision: 1847863

URL: http://svn.apache.org/viewvc?rev=1847863&view=rev
Log:
Add basic auth security to ElasticSearch connector.  Committed on behalf of Steph van Schalkwyk.

Modified:
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_fr_FR.properties
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchAction.java
Fri Nov 30 23:34:01 2018
@@ -26,7 +26,7 @@ import org.apache.http.client.HttpClient
 import java.io.IOException;
 import java.util.Locale;
 
-import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
+//import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
 import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
 import org.apache.manifoldcf.crawler.system.Logging;

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java
Fri Nov 30 23:34:01 2018
@@ -35,13 +35,16 @@ public class ElasticSearchConfig extends
   /** Parameters used for the configuration */
   final private static ParameterEnum[] CONFIGURATIONLIST =
   {
-    ParameterEnum.SERVERVERSION,
+//    ParameterEnum.SERVERVERSION,
     ParameterEnum.SERVERLOCATION,
+    ParameterEnum.USERNAME,
+    ParameterEnum.PASSWORD,
     ParameterEnum.INDEXNAME,
     ParameterEnum.INDEXTYPE,
     ParameterEnum.USEMAPPERATTACHMENTS,
     ParameterEnum.PIPELINENAME,
     ParameterEnum.CONTENTATTRIBUTENAME,
+    ParameterEnum.URIATTRIBUTENAME,
     ParameterEnum.CREATEDDATEATTRIBUTENAME,
     ParameterEnum.MODIFIEDDATEATTRIBUTENAME,
     ParameterEnum.INDEXINGDATEATTRIBUTENAME,
@@ -59,9 +62,19 @@ public class ElasticSearchConfig extends
     super(CONFIGURATIONLIST);
     for (ParameterEnum param : CONFIGURATIONLIST)
     {
-      String value = params.getParameter(param.name());
+      final boolean isPassword = param.name().endsWith("PASSWORD");
+      String value;
+      if (isPassword) {
+        value = params.getObfuscatedParameter(param.name());
+      }
+      else
+      {
+        value = params.getParameter(param.name());
+      }
       if (value == null)
+      {
         value = param.defaultValue;
+      }
       put(param, value);
     }
   }
@@ -82,9 +95,20 @@ public class ElasticSearchConfig extends
   {
     for (ParameterEnum param : CONFIGURATIONLIST)
     {
-      String p = variableContext.getParameter(param.name().toLowerCase(Locale.ROOT));
+      final String paramName = param.name().toLowerCase(Locale.ROOT);
+      final boolean isPassword = param.name().endsWith("PASSWORD");
+      String p = variableContext.getParameter(paramName);
       if (p != null)
-        parameters.setParameter(param.name(), p);
+      {
+        if (isPassword)
+        {
+          parameters.setObfuscatedParameter(param.name(), p);
+        }
+        else
+        {
+          parameters.setParameter(param.name(), p);
+        }
+      }
     }
 
     String useMapperAttachmentsPresent = variableContext.getParameter("usemapperattachments_present");
@@ -97,40 +121,19 @@ public class ElasticSearchConfig extends
     }
   }
 
-  final public boolean isServerAfter5()
+  final public String getServerLocation()
   {
-    return getMajorVersion() >= 5;
-  }
-      
-  final public int getMajorVersion() {
-    final String version = getServerVersion();
-    if (version == null || version.length() == 0) {
-      return 0;
-    }
-    final int index = version.indexOf(".");
-    final String upperVersion;
-    if (index == -1) {
-      upperVersion = version;
-    } else {
-      upperVersion = version.substring(0, index);
-    }
-    int majorVersion;
-    try {
-      majorVersion = Integer.parseInt(upperVersion);
-    } catch (Exception e) {
-      majorVersion = 0;
-    }
-    return majorVersion;
-  }
+    return get(ParameterEnum.SERVERLOCATION);
+  }  
   
-  final public String getServerVersion()
+  final public String getUserName()
   {
-    return get(ParameterEnum.SERVERVERSION);
+    return get(ParameterEnum.USERNAME);
   }
   
-  final public String getServerLocation()
+  final public String getPassword()
   {
-    return get(ParameterEnum.SERVERLOCATION);
+    return get(ParameterEnum.PASSWORD);
   }
 
   final public String getIndexName()
@@ -158,6 +161,11 @@ public class ElasticSearchConfig extends
     return get(ParameterEnum.CONTENTATTRIBUTENAME);
   }
 
+  final public String getUriAttributeName()
+  {
+    return get(ParameterEnum.URIATTRIBUTENAME);
+  }
+
   final public String getCreatedDateAttributeName()
   {
     return get(ParameterEnum.CREATEDDATEATTRIBUTENAME);

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnection.java
Fri Nov 30 23:34:01 2018
@@ -40,21 +40,30 @@ import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.Header;
 import org.apache.http.ProtocolVersion;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.Credentials;
+import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.util.EntityUtils;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.entity.ContentType;
-
+import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.impl.client.BasicAuthCache;
+import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.client.RedirectException;
+import org.apache.http.client.AuthCache;
 import org.apache.http.client.CircularRedirectException;
+import org.apache.http.client.CredentialsProvider;
 import org.apache.http.NoHttpResponseException;
 import org.apache.http.HttpException;
+import org.apache.http.HttpHost;
 import org.apache.http.ParseException;
 
 import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
 import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
 import org.apache.manifoldcf.core.util.URLEncoder;
+import org.apache.manifoldcf.crawler.system.Logging;
 
 public class ElasticSearchConnection
 {
@@ -66,6 +75,10 @@ public class ElasticSearchConnection
 
   private String indexName;
 
+  private String userName;
+
+  private String userPassword;
+
   private String resultDescription;
 
   private String callUrlSnippet;
@@ -93,18 +106,38 @@ public class ElasticSearchConnection
     callUrlSnippet = null;
     serverLocation = config.getServerLocation();
     indexName = config.getIndexName();
+    userName = config.getUserName();
+    userPassword = config.getPassword();
   }
 
-
   protected StringBuffer getApiUrl(String command, boolean checkConnection) throws ManifoldCFException
   {
-    StringBuffer url = new StringBuffer(serverLocation);
+    String basicAuth = "";
+    if (userName.length() > 0 && userPassword.length() >0) {
+      basicAuth = userName + ":" + userPassword + "@";
+    }
+    Logging.connectors.debug("Auth: " + basicAuth);
+
+    String[] serverLocationSplit = serverLocation.split("://",2);
+    StringBuffer url;
+    if (basicAuth.length() > 0) {
+      url = new StringBuffer(serverLocationSplit[0] + "://" + basicAuth + serverLocationSplit[1]);
+    }
+    else {
+      url = new StringBuffer(serverLocationSplit[0] + "://" + basicAuth + serverLocationSplit[1]);
+    }
     if (!serverLocation.endsWith("/"))
       url.append('/');
+
+      Logging.connectors.debug("Url: " + url);
+
     if(!checkConnection)
       url.append(URLEncoder.encode(indexName)).append("/");
     url.append(command);
     callUrlSnippet = url.toString();
+
+    Logging.connectors.debug("UrlEnc: " + url);
+
     return url;
   }
 

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConnector.java
Fri Nov 30 23:34:01 2018
@@ -19,7 +19,7 @@
 
 package org.apache.manifoldcf.agents.output.elasticsearch;
 
-import java.io.File;
+//import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
@@ -27,26 +27,26 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Iterator;
 import java.util.HashMap;
-import java.util.concurrent.TimeUnit;
+//import java.util.concurrent.TimeUnit;
 
 import org.apache.http.conn.HttpClientConnectionManager;
 import org.apache.http.client.HttpClient;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
-import org.apache.http.impl.client.HttpClientBuilder;
+//import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.protocol.HttpRequestExecutor;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.CredentialsProvider;
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.config.SocketConfig;
-import org.apache.http.client.HttpRequestRetryHandler;
-import org.apache.http.protocol.HttpContext;
+//import org.apache.http.client.HttpRequestRetryHandler;
+//import org.apache.http.protocol.HttpContext;
 
-import org.apache.commons.io.FilenameUtils;
+//import org.apache.commons.io.FilenameUtils;
 import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
-import org.apache.manifoldcf.agents.interfaces.IOutputNotifyActivity;
+//import org.apache.manifoldcf.agents.interfaces.IOutputNotifyActivity;
 import org.apache.manifoldcf.agents.interfaces.IOutputRemoveActivity;
-import org.apache.manifoldcf.agents.interfaces.IOutputCheckActivity;
+//import org.apache.manifoldcf.agents.interfaces.IOutputCheckActivity;
 import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
 import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
 import org.apache.manifoldcf.agents.output.BaseOutputConnector;
@@ -54,12 +54,12 @@ import org.apache.manifoldcf.agents.outp
 import org.apache.manifoldcf.agents.output.elasticsearch.ElasticSearchConnection.Result;
 import org.apache.manifoldcf.core.interfaces.Specification;
 import org.apache.manifoldcf.core.interfaces.ConfigParams;
-import org.apache.manifoldcf.core.interfaces.ConfigurationNode;
+//import org.apache.manifoldcf.core.interfaces.ConfigurationNode;
 import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
 import org.apache.manifoldcf.core.interfaces.IPostParameters;
 import org.apache.manifoldcf.core.interfaces.IThreadContext;
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
-import org.apache.manifoldcf.core.interfaces.SpecificationNode;
+//import org.apache.manifoldcf.core.interfaces.SpecificationNode;
 import org.apache.manifoldcf.core.interfaces.VersionContext;
 
 /**
@@ -72,11 +72,9 @@ public class ElasticSearchConnector exte
 
   private final static String ELASTICSEARCH_INDEXATION_ACTIVITY = "Indexation";
   private final static String ELASTICSEARCH_DELETION_ACTIVITY = "Deletion";
-  private final static String ELASTICSEARCH_OPTIMIZE_ACTIVITY = "Optimize";
 
   private final static String[] ELASTICSEARCH_ACTIVITIES =
-  { ELASTICSEARCH_INDEXATION_ACTIVITY, ELASTICSEARCH_DELETION_ACTIVITY,
-      ELASTICSEARCH_OPTIMIZE_ACTIVITY };
+  { ELASTICSEARCH_INDEXATION_ACTIVITY, ELASTICSEARCH_DELETION_ACTIVITY };
 
   private final static String ELASTICSEARCH_TAB_PARAMETERS = "ElasticSearchConnector.Parameters";
 
@@ -210,7 +208,7 @@ public class ElasticSearchConnector exte
   {
     Map<String,String> paramMap = null;
     if (params != null) {
-      paramMap = params.buildMap();
+      paramMap = params.buildMap(out);
       if (tabName != null) {
         paramMap.put("TabName", tabName);
       }
@@ -423,28 +421,4 @@ public class ElasticSearchConnector exte
     }
   }
 
-  @Override
-  public void noteJobComplete(IOutputNotifyActivity activities)
-      throws ManifoldCFException, ServiceInterruption
-  {
-    ElasticSearchConfig config = getConfigParameters(null);
-    HttpClient client = getSession();
-    long startTime = System.currentTimeMillis();
-    ElasticSearchAction oo = new ElasticSearchAction(client, config);
-    try
-    {
-      if (config.isServerAfter5()) {
-        oo.executePOST(CommandEnum._forcemerge, false);
-      } else {
-        oo.executeGET(CommandEnum._optimize, false);
-      }
-    }
-    finally
-    {
-      activities.recordActivity(startTime, ELASTICSEARCH_OPTIMIZE_ACTIVITY, null,
-          oo.getCallUrlSnippet(), oo.getResultCode(),
-          oo.getResultDescription());
-    }
-  }
-
 }

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchDelete.java
Fri Nov 30 23:34:01 2018
@@ -19,15 +19,18 @@
 
 package org.apache.manifoldcf.agents.output.elasticsearch;
 
+import org.apache.http.Header;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpDelete;
-
-import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
+import org.apache.manifoldcf.core.common.Base64;
+//import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
 import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
 import org.apache.manifoldcf.core.util.URLEncoder;
 import org.apache.manifoldcf.crawler.system.Logging;
 
+//import java.util.Locale;
+
 public class ElasticSearchDelete extends ElasticSearchConnection
 {
 
@@ -43,6 +46,16 @@ public class ElasticSearchDelete extends
       HttpDelete method = new HttpDelete(config.getServerLocation() +
           "/" + config.getIndexName() + "/" + config.getIndexType()
           + "/" + idField);
+
+      if (config.getUserName().length() > 0 && config.getPassword().length() >0)
{
+        byte[] basicAuth = (config.getUserName() + ":" + config.getPassword()).getBytes();
+        Base64 encoder = new Base64();
+        String encoding = encoder.encodeByteArray(basicAuth);
+        method.setHeader("Authorization", "Basic " + encoding);
+        Header header = method.getLastHeader("Authorization");
+        Logging.connectors.debug("Header: " + new String(header.getValue().getBytes()));
+      }
+
       call(method);
       String error = checkJson(jsonException);
       if (getResult() == Result.OK && error == null)

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
Fri Nov 30 23:34:01 2018
@@ -32,6 +32,7 @@ import java.util.Iterator;
 import java.text.SimpleDateFormat;
 import java.util.Locale;
 import java.util.TimeZone;
+
 import java.util.Date;
 
 import org.apache.http.client.HttpClient;
@@ -42,10 +43,10 @@ import org.apache.http.util.EntityUtils;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.Header;
 import org.apache.commons.io.IOUtils;
-import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
+//import org.apache.manifoldcf.agents.interfaces.IOutputHistoryActivity;
 import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
 import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
-import org.apache.manifoldcf.agents.output.elasticsearch.ElasticSearchConnection.Result;
+//import org.apache.manifoldcf.agents.output.elasticsearch.ElasticSearchConnection.Result;
 import org.apache.manifoldcf.core.common.Base64;
 import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
 import org.apache.manifoldcf.core.util.URLEncoder;
@@ -159,7 +160,6 @@ public class ElasticSearchIndex extends
             needComma = writeField(pw, needComma, fieldName, fieldValues);
           }
         }
-
         // Standard document fields
         final Date createdDate = document.getCreatedDate();
         if (createdDate != null && createdDateAttributeName != null && createdDateAttributeName.length()
> 0)
@@ -181,7 +181,6 @@ public class ElasticSearchIndex extends
         {
           needComma = writeField(pw, needComma, mimeTypeAttributeName, new String[]{mimeType});
         }
-        
         needComma = writeACLs(pw, needComma, "document", acls, denyAcls);
         needComma = writeACLs(pw, needComma, "share", shareAcls, shareDenyAcls);
         needComma = writeACLs(pw, needComma, "parent", parentAcls, parentDenyAcls);
@@ -245,7 +244,6 @@ public class ElasticSearchIndex extends
                 }
               }
             }
-            
             pw.append("\"");
             needComma = true;
           }
@@ -420,16 +418,13 @@ public class ElasticSearchIndex extends
     String[] acls, String[] denyAcls, String[] shareAcls, String[] shareDenyAcls, String[]
parentAcls, String[] parentDenyAcls)
     throws ManifoldCFException, ServiceInterruption
   {
-
-
     final String idField = URLEncoder.encode(documentURI);
     final String encodedPipelineName = (config.getPipelineName() == null || config.getPipelineName().length()
== 0)?null:URLEncoder.encode(config.getPipelineName());
-    
-    final String command = config.getIndexType() + "/" + idField;
+    final String command = config.getIndexType() + "/"  + idField;
     final String fullCommand = (encodedPipelineName == null)?command:(command + "?pipeline="
+ encodedPipelineName);
-    
     StringBuffer url = getApiUrl(fullCommand, false);
     HttpPut put = new HttpPut(url.toString());
+    Logging.connectors.debug("HttPutUri: " + url.toString());
     put.setEntity(new IndexRequestEntity(document, inputStream,
       acls, denyAcls, shareAcls, shareDenyAcls, parentAcls, parentDenyAcls,
       config.getUseMapperAttachments(),
@@ -438,13 +433,23 @@ public class ElasticSearchIndex extends
       config.getModifiedDateAttributeName(),
       config.getIndexingDateAttributeName(),
       config.getMimeTypeAttributeName()));
+    
+      if (config.getUserName().length() > 0 && config.getPassword().length() >0)
{
+        byte[] basicAuth = (config.getUserName() + ":" + config.getPassword()).getBytes();
+        Base64 encoder = new Base64();
+        String encoding = encoder.encodeByteArray(basicAuth);
+        put.setHeader("Authorization", "Basic " + encoding);
+        Header header = put.getLastHeader("Authorization");
+        Logging.connectors.debug("Header: " + new String(header.getValue().getBytes()));
+      }
+
     if (call(put) == false)
       return false;
     String error = checkJson(jsonException);
     if (getResult() == Result.OK && error == null)
       return true;
     setResult("JSONERROR",Result.ERROR, error);
-    Logging.connectors.warn("ES: Index failed: "+getResponse());
+    Logging.connectors.warn("ES: Index failed: " + getResponse());
     return true;
   }
 

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java
Fri Nov 30 23:34:01 2018
@@ -22,6 +22,8 @@ package org.apache.manifoldcf.agents.out
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
+
 import org.apache.manifoldcf.agents.output.elasticsearch.ElasticSearchParam.ParameterEnum;
 
 /** 
@@ -33,27 +35,31 @@ public class ElasticSearchParam extends
   /** Parameters constants */
   public enum ParameterEnum
   {
-    SERVERVERSION(""),
-    
     SERVERLOCATION("http://localhost:9200/"),
 
     INDEXNAME("index"),
 
-    INDEXTYPE("generictype"),
+    USERNAME(""),
+
+    PASSWORD(""),
+
+    INDEXTYPE("generic"),
 
-    USEMAPPERATTACHMENTS("true"),
+    USEMAPPERATTACHMENTS("false"),
     
     PIPELINENAME(""),
 
-    CONTENTATTRIBUTENAME(""),
+    CONTENTATTRIBUTENAME("content"),
 
-    CREATEDDATEATTRIBUTENAME(""),
+    URIATTRIBUTENAME("url"),
+
+    CREATEDDATEATTRIBUTENAME("created"),
     
-    MODIFIEDDATEATTRIBUTENAME(""),
+    MODIFIEDDATEATTRIBUTENAME("last-modified"),
     
-    INDEXINGDATEATTRIBUTENAME(""),
+    INDEXINGDATEATTRIBUTENAME("indexed"),
     
-    MIMETYPEATTRIBUTENAME(""),
+    MIMETYPEATTRIBUTENAME("mime-type"),
     
     FIELDLIST("");
 
@@ -72,11 +78,23 @@ public class ElasticSearchParam extends
     super(params.length);
   }
 
-  final public Map<String, String> buildMap()
+  final public Map<String, String> buildMap(IHTTPOutput out)
   {
     Map<String, String> rval = new HashMap<String, String>();
     for (Map.Entry<ParameterEnum, String> entry : this.entrySet())
-      rval.put(entry.getKey().name(), entry.getValue());
+    {
+      final String key = entry.getKey().name();
+      final boolean isPassword = key.endsWith("PASSWORD");
+      if (isPassword) 
+      {
+        // Do not put passwords in plain text in forms
+        rval.put(key, out.mapPasswordToKey(entry.getValue()));
+      }
+      else
+      {
+        rval.put(key, entry.getValue());
+      }
+    }
     return rval;
   }
 

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties
Fri Nov 30 23:34:01 2018
@@ -15,9 +15,11 @@
 
 ElasticSearchConnector.Parameters=Parameters
 
-ElasticSearchConnector.ServerVersion=Server version:
+#ElasticSearchConnector.ServerVersion=Server version:
 ElasticSearchConnector.ServerLocation=Server location
 ElasticSearchConnector.URLColon=(URL):
+ElasticSearchConnector.UserNameColon=User name:
+ElasticSearchConnector.PasswordColon=User password:
 ElasticSearchConnector.IndexNameColon=Index name:
 ElasticSearchConnector.IndexTypeColon=Index type:
 ElasticSearchConnector.UseMapperAttachmentsColon=Use mapper-attachments:

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties
Fri Nov 30 23:34:01 2018
@@ -15,9 +15,11 @@
 
 ElasticSearchConnector.Parameters=Parámetros
 
-ElasticSearchConnector.ServerVersion=Server version:
+#ElasticSearchConnector.ServerVersion=Server version:
 ElasticSearchConnector.ServerLocation=Ubicación del servidor
 ElasticSearchConnector.URLColon=(URL):
+ElasticSearchConnector.UserNameColon=User name:
+ElasticSearchConnector.PasswordColon=User password:
 ElasticSearchConnector.IndexNameColon=El nombre de índice:
 ElasticSearchConnector.IndexTypeColon=tipo de índice:
 ElasticSearchConnector.UseMapperAttachmentsColon=Use mapper-attachments:

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_fr_FR.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_fr_FR.properties?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_fr_FR.properties
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_fr_FR.properties
Fri Nov 30 23:34:01 2018
@@ -17,6 +17,8 @@ ElasticSearchConnector.Parameters=Param�
 
 ElasticSearchConnector.ServerLocation=Emplacement du serveur
 ElasticSearchConnector.URLColon=(URL):
+ElasticSearchConnector.UserNameColon=User name:
+ElasticSearchConnector.PasswordColon=User password:
 ElasticSearchConnector.IndexNameColon=Nom de l'index:
 ElasticSearchConnector.IndexTypeColon=Type d'index:
 
@@ -28,4 +30,6 @@ ElasticSearchConnector.ElasticSearch=Ela
 
 ElasticSearchConnector.MaxFileSizeBytesColon=Taille max des fichiers (octets):
 ElasticSearchConnector.AllowedMIMETypesColon=Types MIME admis:
-ElasticSearchConnector.AllowedFileExtensionsColon=Extensions de fichiers admises\:
\ No newline at end of file
+ElasticSearchConnector.AllowedFileExtensionsColon=Extensions de fichiers admises\:
+
+ElasticSearchConnector.UriAttributeNameColon=Uri field name:

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties
Fri Nov 30 23:34:01 2018
@@ -15,9 +15,11 @@
 
 ElasticSearchConnector.Parameters=引数
 
-ElasticSearchConnector.ServerVersion=Server version:
+#ElasticSearchConnector.ServerVersion=Server version:
 ElasticSearchConnector.ServerLocation=サーバ
 ElasticSearchConnector.URLColon=(URL):
+ElasticSearchConnector.UserNameColon=User name:
+ElasticSearchConnector.PasswordColon=User password:
 ElasticSearchConnector.IndexNameColon=インデックス名:
 ElasticSearchConnector.IndexTypeColon=タイプ名:
 ElasticSearchConnector.UseMapperAttachmentsColon=Use mapper-attachments:

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties
Fri Nov 30 23:34:01 2018
@@ -15,9 +15,11 @@
 
 ElasticSearchConnector.Parameters=参数
 
-ElasticSearchConnector.ServerVersion=Server version:
+#ElasticSearchConnector.ServerVersion=Server version:
 ElasticSearchConnector.ServerLocation=服务器
 ElasticSearchConnector.URLColon=(URL):
+ElasticSearchConnector.UserNameColon=User name:
+ElasticSearchConnector.PasswordColon=User password:
 ElasticSearchConnector.IndexNameColon=索引名: 
 ElasticSearchConnector.IndexTypeColon=索引类型: 
 ElasticSearchConnector.UseMapperAttachmentsColon=Use mapper-attachments:

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html
Fri Nov 30 23:34:01 2018
@@ -20,32 +20,33 @@
 <table class="displaytable">
   <tr>
     <td class="description">
-      <nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.ServerVersion'))</nobr>
+      <nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.ServerLocation'))</nobr>
+        $Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.URLColon'))
     </td>
-    <td class="value"><input name="serverversion" type="text"
-      value="$Encoder.attributeEscape($SERVERVERSION)" size="10" />
+    <td class="value"><input name="serverlocation" type="text"
+      value="$Encoder.attributeEscape($SERVERLOCATION)" size="48" />
     </td>
   </tr>
   <tr>
     <td class="description">
-      <nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.ServerLocation'))</nobr>
-        $Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.URLColon'))
+      <nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.UserNameColon'))</nobr></td>
+    <td class="value"><input name="username" type="text" value="$Encoder.attributeEscape($USERNAME)"
size="24" />
     </td>
-    <td class="value"><input name="serverlocation" type="text"
-      value="$Encoder.attributeEscape($SERVERLOCATION)" size="48" />
+  </tr>
+  <tr>
+    <td class="description">
+      <nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.PasswordColon'))</nobr></td>
+    <td class="value"><input name="password" type="password" value="$Encoder.attributeEscape($PASSWORD)"
size="24" />
     </td>
   </tr>
   <tr>
     <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.IndexNameColon'))</nobr></td>
-    <td class="value"><input name="indexname" type="text" value="$Encoder.attributeEscape($INDEXNAME)"
-      size="24" /></td>
+    <td class="value"><input name="indexname" type="text" value="$Encoder.attributeEscape($INDEXNAME)"
size="24" /></td>
   </tr>
   <tr>
     <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.IndexTypeColon'))</nobr></td>
-    <td class="value"><input name="indextype" type="text" value="$Encoder.attributeEscape($INDEXTYPE)"
-      size="24" /></td>
+    <td class="value"><input name="indextype" type="text" value="$Encoder.attributeEscape($INDEXTYPE)"
size="24" /></td>
   </tr>
-
   <tr>
     <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.UseMapperAttachmentsColon'))</nobr></td>
     <td class="value">
@@ -63,7 +64,6 @@
     <td class="value"><input name="pipelinename" type="text" value="$Encoder.attributeEscape($PIPELINENAME)"
       size="24" /></td>
   </tr>
-
   <tr>
     <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.ContentAttributeNameColon'))</nobr></td>
     <td class="value"><input name="contentattributename" type="text" value="$Encoder.attributeEscape($CONTENTATTRIBUTENAME)"
@@ -93,8 +93,6 @@
 </table>
 
 #else
-
-<input type="hidden" name="serverversion" value="$Encoder.attributeEscape($SERVERVERSION)"
/>
 <input type="hidden" name="serverlocation" value="$Encoder.attributeEscape($SERVERLOCATION)"
/>
 <input type="hidden" name="indexname" value="$Encoder.attributeEscape($INDEXNAME)" />
 <input type="hidden" name="indextype" value="$Encoder.attributeEscape($INDEXTYPE)" />
@@ -103,6 +101,7 @@
 <input type="hidden" name="usemapperattachments" value="$Encoder.bodyEscape($USEMAPPERATTACHMENTS)"
/>
 <input type="hidden" name="pipelinename" value="$Encoder.attributeEscape($PIPELINENAME)"
/>
 <input type="hidden" name="contentattributename" value="$Encoder.attributeEscape($CONTENTATTRIBUTENAME)"
/>
+<input type="hidden" name="uriattributename" value="$Encoder.attributeEscape($URIATTRIBUTENAME)"
/>
 <input type="hidden" name="createddateattributename" value="$Encoder.attributeEscape($CREATEDDATEATTRIBUTENAME)"
/>
 <input type="hidden" name="modifieddateattributename" value="$Encoder.attributeEscape($MODIFIEDDATEATTRIBUTENAME)"
/>
 <input type="hidden" name="indexingdateattributename" value="$Encoder.attributeEscape($INDEXINGDATEATTRIBUTENAME)"
/>

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html?rev=1847863&r1=1847862&r2=1847863&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html
Fri Nov 30 23:34:01 2018
@@ -17,15 +17,19 @@
 
 <table class="displaytable">
   <tr>
-    <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.ServerVersion'))</nobr>
-    <td class="value">$Encoder.bodyEscape($SERVERVERSION)</td>
-  </tr>
-  <tr>
     <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.ServerLocation'))</nobr>
     $Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.URLColon'))</td>
     <td class="value">$Encoder.bodyEscape($SERVERLOCATION)</td>
   </tr>
-    <tr>
+  <tr>
+    <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.UserNameColon'))</nobr></td>
+    <td class="value">$Encoder.bodyEscape($USERNAME)</td>
+  </tr>
+  <tr>
+    <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.PasswordColon'))</nobr></td>
+    <td class="value">*******</td>
+  </tr>
+  <tr>
     <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.IndexNameColon'))</nobr></td>
     <td class="value">$Encoder.bodyEscape($INDEXNAME)</td>
   </tr>
@@ -61,5 +65,4 @@
     <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.MimeTypeAttributeNameColon'))</nobr></td>
     <td class="value">$Encoder.bodyEscape($MIMETYPEATTRIBUTENAME)</td>
   </tr>
-
 </table>
\ No newline at end of file



Mime
View raw message