manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1799511 - in /manifoldcf/trunk: ./ connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsear...
Date Wed, 21 Jun 2017 20:43:40 GMT
Author: kwright
Date: Wed Jun 21 20:43:40 2017
New Revision: 1799511

URL: http://svn.apache.org/viewvc?rev=1799511&view=rev
Log:
Tentative fix for CONNECTORS-1433

Modified:
    manifoldcf/trunk/CHANGES.txt
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html
    manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html
    manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java

Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Wed Jun 21 20:43:40 2017
@@ -3,6 +3,9 @@ $Id$
 
 ======================= 2.8-dev =====================
 
+CONNECTORS-1433: Add pipeline specification to ES connector.
+(Steph van Schalkwyk, Karl Wright)
+
 CONNECTORS-1432: Fix deleting stale check methods from window object
 (Steph van Schalkwyk, Karl Wright, Kishore Kumar)
 

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchConfig.java
Wed Jun 21 20:43:40 2017
@@ -40,6 +40,7 @@ public class ElasticSearchConfig extends
     ParameterEnum.INDEXNAME,
     ParameterEnum.INDEXTYPE,
     ParameterEnum.USEMAPPERATTACHMENTS,
+    ParameterEnum.PIPELINENAME,
     ParameterEnum.CONTENTATTRIBUTENAME,
     ParameterEnum.CREATEDDATEATTRIBUTENAME,
     ParameterEnum.MODIFIEDDATEATTRIBUTENAME,
@@ -147,6 +148,11 @@ public class ElasticSearchConfig extends
     return Boolean.valueOf(get(ParameterEnum.USEMAPPERATTACHMENTS));
   }
 
+  final public String getPipelineName()
+  {
+    return get(ParameterEnum.PIPELINENAME);
+  }
+  
   final public String getContentAttributeName()
   {
     return get(ParameterEnum.CONTENTATTRIBUTENAME);

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchIndex.java
Wed Jun 21 20:43:40 2017
@@ -392,11 +392,15 @@ public class ElasticSearchIndex extends
     String[] acls, String[] denyAcls, String[] shareAcls, String[] shareDenyAcls, String[]
parentAcls, String[] parentDenyAcls)
     throws ManifoldCFException, ServiceInterruption
   {
-    String idField;
 
-    idField = URLEncoder.encode(documentURI);
 
-    StringBuffer url = getApiUrl(config.getIndexType() + "/" + idField, false);
+    final String idField = URLEncoder.encode(documentURI);
+    final String encodedPipelineName = (config.getPipelineName() == null || config.getPipelineName().length()
== 0)?null:URLEncoder.encode(config.getPipelineName());
+    
+    final String command = config.getIndexType() + "/" + idField;
+    final String fullCommand = (encodedPipelineName == null)?command:(command + "?pipeline="
+ encodedPipelineName);
+    
+    StringBuffer url = getApiUrl(fullCommand, false);
     HttpPut put = new HttpPut(url.toString());
     put.setEntity(new IndexRequestEntity(document, inputStream,
       acls, denyAcls, shareAcls, shareDenyAcls, parentAcls, parentDenyAcls,

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/java/org/apache/manifoldcf/agents/output/elasticsearch/ElasticSearchParam.java
Wed Jun 21 20:43:40 2017
@@ -42,6 +42,8 @@ public class ElasticSearchParam extends
     INDEXTYPE("generictype"),
 
     USEMAPPERATTACHMENTS("true"),
+    
+    PIPELINENAME(""),
 
     CONTENTATTRIBUTENAME(""),
 

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_en_US.properties
Wed Jun 21 20:43:40 2017
@@ -21,6 +21,7 @@ ElasticSearchConnector.URLColon=(URL):
 ElasticSearchConnector.IndexNameColon=Index name:
 ElasticSearchConnector.IndexTypeColon=Index type:
 ElasticSearchConnector.UseMapperAttachmentsColon=Use mapper-attachments:
+ElasticSearchConnector.PipelineNameColon=Pipeline name:
 ElasticSearchConnector.ContentAttributeNameColon=Content field name:
 ElasticSearchConnector.CreatedDateAttributeNameColon=Created date field name:
 ElasticSearchConnector.ModifiedDateAttributeNameColon=Modified date field name:

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_es_ES.properties
Wed Jun 21 20:43:40 2017
@@ -21,6 +21,7 @@ ElasticSearchConnector.URLColon=(URL):
 ElasticSearchConnector.IndexNameColon=El nombre de índice:
 ElasticSearchConnector.IndexTypeColon=tipo de índice:
 ElasticSearchConnector.UseMapperAttachmentsColon=Use mapper-attachments:
+ElasticSearchConnector.PipelineNameColon=Pipeline name:
 ElasticSearchConnector.ContentAttributeNameColon=Content field name:
 ElasticSearchConnector.CreatedDateAttributeNameColon=Created date field name:
 ElasticSearchConnector.ModifiedDateAttributeNameColon=Modified date field name:

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_ja_JP.properties
Wed Jun 21 20:43:40 2017
@@ -21,6 +21,7 @@ ElasticSearchConnector.URLColon=(URL):
 ElasticSearchConnector.IndexNameColon=インデックス名:
 ElasticSearchConnector.IndexTypeColon=タイプ名:
 ElasticSearchConnector.UseMapperAttachmentsColon=Use mapper-attachments:
+ElasticSearchConnector.PipelineNameColon=Pipeline name:
 ElasticSearchConnector.ContentAttributeNameColon=Content field name:
 ElasticSearchConnector.CreatedDateAttributeNameColon=Created date field name:
 ElasticSearchConnector.ModifiedDateAttributeNameColon=Modified date field name:

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/elasticsearch/common_zh_CN.properties
Wed Jun 21 20:43:40 2017
@@ -21,6 +21,7 @@ ElasticSearchConnector.URLColon=(URL):
 ElasticSearchConnector.IndexNameColon=索引名: 
 ElasticSearchConnector.IndexTypeColon=索引类型: 
 ElasticSearchConnector.UseMapperAttachmentsColon=Use mapper-attachments:
+ElasticSearchConnector.PipelineNameColon=Pipeline name:
 ElasticSearchConnector.ContentAttributeNameColon=Content field name:
 ElasticSearchConnector.CreatedDateAttributeNameColon=Created date field name:
 ElasticSearchConnector.ModifiedDateAttributeNameColon=Modified date field name:

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/editConfiguration_Parameters.html
Wed Jun 21 20:43:40 2017
@@ -58,7 +58,12 @@
   #end
     </td>
   </tr>
-  
+  <tr>
+    <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.PipelineNameColon'))</nobr></td>
+    <td class="value"><input name="pipelinename" type="text" value="$Encoder.attributeEscape($PIPELINENAME)"
+      size="24" /></td>
+  </tr>
+
   <tr>
     <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.ContentAttributeNameColon'))</nobr></td>
     <td class="value"><input name="contentattributename" type="text" value="$Encoder.attributeEscape($CONTENTATTRIBUTENAME)"
@@ -96,6 +101,7 @@
 <input type="hidden" name="usemapperattachments_checkbox" value="false"/>
 <input type="hidden" name="usemapperattachments_present" value="true"/>
 <input type="hidden" name="usemapperattachments" value="$Encoder.bodyEscape($USEMAPPERATTACHMENTS)"
/>
+<input type="hidden" name="pipelinename" value="$Encoder.attributeEscape($PIPELINENAME)"
/>
 <input type="hidden" name="contentattributename" value="$Encoder.attributeEscape($CONTENTATTRIBUTENAME)"
/>
 <input type="hidden" name="createddateattributename" value="$Encoder.attributeEscape($CREATEDDATEATTRIBUTENAME)"
/>
 <input type="hidden" name="modifieddateattributename" value="$Encoder.attributeEscape($MODIFIEDDATEATTRIBUTENAME)"
/>

Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html
(original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/main/resources/org/apache/manifoldcf/agents/output/elasticsearch/viewConfiguration.html
Wed Jun 21 20:43:40 2017
@@ -38,6 +38,10 @@
     <td class="value"><nobr>$Encoder.bodyEscape($USEMAPPERATTACHMENTS)</nobr></td>
   </tr>
   <tr>
+    <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.PipelineNameColon'))</nobr></td>
+    <td class="value"><nobr>$Encoder.bodyEscape($PIPELINENAME)</nobr></td>
+  </tr>
+  <tr>
     <td class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('ElasticSearchConnector.ContentAttributeNameColon'))</nobr></td>
     <td class="value">$Encoder.bodyEscape($CONTENTATTRIBUTENAME)</td>
   </tr>

Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java?rev=1799511&r1=1799510&r2=1799511&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java
(original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java
Wed Jun 21 20:43:40 2017
@@ -193,11 +193,11 @@ public class ModifiedHttpSolrClient exte
             if (name == null) {
               name = "";
             }
-            parts.add(new FormBodyPart(name,
+            parts.add(new FormBodyPart(encodeForHeader(name),
                 new InputStreamBody(
                     content.getStream(),
                     contentType,
-                    content.getName())));
+                    encodeForHeader(content.getName()))));
           }
         }
 
@@ -293,4 +293,21 @@ public class ModifiedHttpSolrClient exte
     return sb.toString();
   }
   
+  // This is a hack added by KDW on 6/21/2017 because HttpClient doesn't do any character
+  // escaping when it puts together header and file names
+  private static String encodeForHeader(final String headerName) {
+    if (headerName == null) {
+      return null;
+    }
+    final StringBuilder sb = new StringBuilder();
+    for (int i = 0; i < headerName.length(); i++) {
+      final char x = headerName.charAt(i);
+      if (x == '"' || x == '\\') {
+        sb.append("\\");
+      }
+      sb.append(x);
+    }
+    return sb.toString();
+  }
+  
 }



Mime
View raw message