manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1841843 - in /manifoldcf/branches/release-2.11-branch: ./ connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/
Date Mon, 24 Sep 2018 13:04:35 GMT
Author: kwright
Date: Mon Sep 24 13:04:34 2018
New Revision: 1841843

URL: http://svn.apache.org/viewvc?rev=1841843&view=rev
Log:
Pull up fix for CONNECTORS-1533.

Modified:
    manifoldcf/branches/release-2.11-branch/   (props changed)
    manifoldcf/branches/release-2.11-branch/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
    manifoldcf/branches/release-2.11-branch/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java

Propchange: manifoldcf/branches/release-2.11-branch/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Sep 24 13:04:34 2018
@@ -152,4 +152,4 @@
 /manifoldcf/branches/CONNECTORS-981:1605049-1605773
 /manifoldcf/branches/CONNECTORS-989:1611600-1612101
 /manifoldcf/branches/CONNECTORS-990:1610284-1610707
-/manifoldcf/trunk:1841397,1841587,1841596,1841598,1841618
+/manifoldcf/trunk:1841397,1841587,1841596,1841598,1841618,1841840

Modified: manifoldcf/branches/release-2.11-branch/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-2.11-branch/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1841843&r1=1841842&r2=1841843&view=diff
==============================================================================
--- manifoldcf/branches/release-2.11-branch/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++ manifoldcf/branches/release-2.11-branch/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Mon Sep 24 13:04:34 2018
@@ -49,6 +49,7 @@ import org.apache.http.auth.UsernamePass
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
 import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
 import org.apache.solr.client.solrj.request.UpdateRequest;
 import org.apache.solr.client.solrj.impl.XMLResponseParser;

Modified: manifoldcf/branches/release-2.11-branch/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-2.11-branch/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java?rev=1841843&r1=1841842&r2=1841843&view=diff
==============================================================================
--- manifoldcf/branches/release-2.11-branch/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java
(original)
+++ manifoldcf/branches/release-2.11-branch/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java
Mon Sep 24 13:04:34 2018
@@ -36,6 +36,7 @@ import org.apache.http.NameValuePair;
 import org.apache.http.NoHttpResponseException;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpDelete;
 import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
@@ -44,9 +45,12 @@ import org.apache.http.client.methods.Ht
 import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.client.params.ClientPNames;
 import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.entity.BasicHttpEntity;
+import org.apache.http.entity.ContentType;
 import org.apache.http.entity.InputStreamEntity;
 import org.apache.http.entity.mime.FormBodyPart;
 import org.apache.http.entity.mime.HttpMultipartMode;
+import org.apache.http.entity.mime.MultipartEntity;
 import org.apache.http.entity.mime.content.InputStreamBody;
 import org.apache.http.entity.mime.content.StringBody;
 import org.apache.http.impl.client.DefaultHttpClient;
@@ -56,11 +60,15 @@ import org.apache.http.util.EntityUtils;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.BasicHttpEntity;
 import org.apache.manifoldcf.core.util.URLDecoder;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
 import org.apache.solr.client.solrj.ResponseParser;
-import org.apache.solr.client.solrj.impl.BinaryResponseParser;
-import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.V2RequestSupport;
+import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.request.V2Request;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.BinaryResponseParser;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.request.RequestWriter;
 import org.apache.solr.client.solrj.request.UpdateRequest;
@@ -103,7 +111,211 @@ public class ModifiedHttpSolrClient exte
     super(baseURL, client, parser, allowCompression);
     httpClient = client;
   }
-  
+
+  @Override
+  protected HttpRequestBase createMethod(SolrRequest request, String collection) throws IOException,
SolrServerException {
+    if (request instanceof V2RequestSupport) {
+      request = ((V2RequestSupport) request).getV2Request();
+    }
+    SolrParams params = request.getParams();
+    RequestWriter.ContentWriter contentWriter = requestWriter.getContentWriter(request);
+    Collection<ContentStream> streams = contentWriter == null ? requestWriter.getContentStreams(request)
: null;
+    String path = requestWriter.getPath(request);
+    if (path == null || !path.startsWith("/")) {
+      path = DEFAULT_PATH;
+    }
+    
+    ResponseParser parser = request.getResponseParser();
+    if (parser == null) {
+      parser = this.parser;
+    }
+    
+    // The parser 'wt=' and 'version=' params are used instead of the original
+    // params
+    ModifiableSolrParams wparams = new ModifiableSolrParams(params);
+    if (parser != null) {
+      wparams.set(CommonParams.WT, parser.getWriterType());
+      wparams.set(CommonParams.VERSION, parser.getVersion());
+    }
+    if (invariantParams != null) {
+      wparams.add(invariantParams);
+    }
+
+    String basePath = baseUrl;
+    if (collection != null)
+      basePath += "/" + collection;
+
+    if (request instanceof V2Request) {
+      if (System.getProperty("solr.v2RealPath") == null) {
+        basePath = baseUrl.replace("/solr", "/api");
+      } else {
+        basePath = baseUrl + "/____v2";
+      }
+    }
+
+    if (SolrRequest.METHOD.GET == request.getMethod()) {
+      if (streams != null || contentWriter != null) {
+        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!");
+      }
+
+      return new HttpGet(basePath + path + toQueryString(wparams, false));
+    }
+
+    if (SolrRequest.METHOD.DELETE == request.getMethod()) {
+      return new HttpDelete(basePath + path + toQueryString(wparams, false));
+    }
+
+    if (SolrRequest.METHOD.POST == request.getMethod() || SolrRequest.METHOD.PUT == request.getMethod())
{
+      
+      // Hack to allow short queries to go one way, and long queries to go another.
+      final boolean mustUseMultipart = this.useMultiPartPost && request instanceof
org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
+      if (mustUseMultipart) {
+        //System.out.println("Overriding with streams");
+        streams = requestWriter.getContentStreams(request);
+      }
+
+      //System.out.println("Post or put");
+      String url = basePath + path;
+      boolean hasNullStreamName = false;
+      if (streams != null) {
+        for (ContentStream cs : streams) {
+          if (cs.getName() == null) {
+            hasNullStreamName = true;
+            break;
+          }
+        }
+      }
+      boolean isMultipart = ((this.useMultiPartPost && SolrRequest.METHOD.POST ==
request.getMethod())
+          || (streams != null && streams.size() > 1)) && !hasNullStreamName;
+
+      LinkedList<NameValuePair> postOrPutParams = new LinkedList<>();
+
+      if(contentWriter != null && !mustUseMultipart) {
+        //System.out.println(" using contentwriter");
+        String fullQueryUrl = url + toQueryString(wparams, false);
+        HttpEntityEnclosingRequestBase postOrPut = SolrRequest.METHOD.POST == request.getMethod()
?
+            new HttpPost(fullQueryUrl) : new HttpPut(fullQueryUrl);
+        postOrPut.addHeader("Content-Type",
+            contentWriter.getContentType());
+        postOrPut.setEntity(new BasicHttpEntity(){
+          @Override
+          public boolean isStreaming() {
+            return true;
+          }
+
+          @Override
+          public void writeTo(OutputStream outstream) throws IOException {
+            contentWriter.write(outstream);
+          }
+        });
+        return postOrPut;
+
+      } else if (streams == null || isMultipart) {
+        // send server list and request list as query string params
+        ModifiableSolrParams queryParams = calculateQueryParams(getQueryParams(), wparams);
+        queryParams.add(calculateQueryParams(request.getQueryParams(), wparams));
+        String fullQueryUrl = url + toQueryString(queryParams, false);
+        HttpEntityEnclosingRequestBase postOrPut = fillContentStream(request, streams, wparams,
isMultipart, postOrPutParams, fullQueryUrl);
+        return postOrPut;
+      }
+      // It is has one stream, it is the post body, put the params in the URL
+      else {
+        String fullQueryUrl = url + toQueryString(wparams, false);
+        HttpEntityEnclosingRequestBase postOrPut = SolrRequest.METHOD.POST == request.getMethod()
?
+            new HttpPost(fullQueryUrl) : new HttpPut(fullQueryUrl);
+        fillSingleContentStream(streams, postOrPut);
+
+        return postOrPut;
+      }
+    }
+
+    throw new SolrServerException("Unsupported method: " + request.getMethod());
+
+  }
+
+  private void fillSingleContentStream(Collection<ContentStream> streams, HttpEntityEnclosingRequestBase
postOrPut) throws IOException {
+    // Single stream as body
+    // Using a loop just to get the first one
+    final ContentStream[] contentStream = new ContentStream[1];
+    for (ContentStream content : streams) {
+      contentStream[0] = content;
+      break;
+    }
+    Long size = contentStream[0].getSize();
+    postOrPut.setEntity(new InputStreamEntity(contentStream[0].getStream(), size == null
? -1 : size) {
+      @Override
+      public Header getContentType() {
+        return new BasicHeader("Content-Type", contentStream[0].getContentType());
+      }
+
+      @Override
+      public boolean isRepeatable() {
+        return false;
+      }
+    });
+
+  }
+
+  private HttpEntityEnclosingRequestBase fillContentStream(SolrRequest request, Collection<ContentStream>
streams, ModifiableSolrParams wparams, boolean isMultipart, LinkedList<NameValuePair>
postOrPutParams, String fullQueryUrl) throws IOException {
+    HttpEntityEnclosingRequestBase postOrPut = SolrRequest.METHOD.POST == request.getMethod()
?
+        new HttpPost(fullQueryUrl) : new HttpPut(fullQueryUrl);
+
+    if (!isMultipart) {
+      postOrPut.addHeader("Content-Type",
+          "application/x-www-form-urlencoded; charset=UTF-8");
+    }
+
+    List<FormBodyPart> parts = new LinkedList<>();
+    Iterator<String> iter = wparams.getParameterNamesIterator();
+    while (iter.hasNext()) {
+      String p = iter.next();
+      String[] vals = wparams.getParams(p);
+      if (vals != null) {
+        for (String v : vals) {
+          if (isMultipart) {
+            parts.add(new FormBodyPart(p, new StringBody(v, StandardCharsets.UTF_8)));
+          } else {
+            postOrPutParams.add(new BasicNameValuePair(p, v));
+          }
+        }
+      }
+    }
+
+    // TODO: remove deprecated - first simple attempt failed, see {@link MultipartEntityBuilder}
+    if (isMultipart && streams != null) {
+      for (ContentStream content : streams) {
+        String contentType = content.getContentType();
+        if (contentType == null) {
+          contentType = BinaryResponseParser.BINARY_CONTENT_TYPE; // default
+        }
+        String name = content.getName();
+        if (name == null) {
+          name = "";
+        }
+        parts.add(new FormBodyPart(encodeForHeader(name),
+            new InputStreamBody(
+                content.getStream(),
+                ContentType.parse(contentType),
+                encodeForHeader(content.getName()))));
+      }
+    }
+
+    //System.out.println("Using multipart post!");
+    if (parts.size() > 0) {
+      ModifiedMultipartEntity entity = new ModifiedMultipartEntity(HttpMultipartMode.STRICT,
null, StandardCharsets.UTF_8);
+      //MultipartEntity entity = new MultipartEntity(HttpMultipartMode.STRICT);
+      for (FormBodyPart p : parts) {
+        entity.addPart(p);
+      }
+      postOrPut.setEntity(entity);
+    } else {
+      //not using multipart
+      postOrPut.setEntity(new UrlEncodedFormEntity(postOrPutParams, StandardCharsets.UTF_8));
+    }
+    return postOrPut;
+  }
+
+    /*
   @Override
   protected HttpRequestBase createMethod(final SolrRequest request, String collection) throws
IOException, SolrServerException {
     
@@ -276,7 +488,7 @@ public class ModifiedHttpSolrClient exte
     throw new SolrServerException("Unsupported method: " + request.getMethod());
 
   }
-
+*/
   public static String toQueryString( SolrParams params, boolean xml ) {
     StringBuilder sb = new StringBuilder(128);
     try {



Mime
View raw message