lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tflo...@apache.org
Subject [38/58] [abbrv] lucene-solr:jira/solr-10233: SOLR-10431: Make it possible to invoke v2 api calls using SolrJ
Date Fri, 12 May 2017 23:39:05 GMT
SOLR-10431: Make it possible to invoke v2 api calls using SolrJ


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cc8b5bab
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cc8b5bab
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cc8b5bab

Branch: refs/heads/jira/solr-10233
Commit: cc8b5bab0bdad4d57ac3764e9c3b313cb07fcddc
Parents: 95968c6
Author: Cao Manh Dat <datcm@apache.org>
Authored: Thu May 11 09:06:03 2017 +0700
Committer: Cao Manh Dat <datcm@apache.org>
Committed: Thu May 11 09:06:03 2017 +0700

----------------------------------------------------------------------
 solr/CHANGES.txt                                |   2 +
 .../solr/security/BasicAuthIntegrationTest.java |  15 ++-
 .../solr/client/solrj/impl/CloudSolrClient.java |  23 +++-
 .../solr/client/solrj/impl/HttpSolrClient.java  |  14 ++
 .../solr/client/solrj/request/V2Request.java    | 127 +++++++++++++++++++
 .../apache/solr/common/params/CommonParams.java |   4 +-
 .../client/solrj/request/TestV2Request.java     |  89 +++++++++++++
 .../java/org/apache/solr/SolrTestCaseJ4.java    |   2 +
 8 files changed, 265 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cc8b5bab/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index b7aaa8b..530a065 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -90,6 +90,8 @@ New Features
 
 * SOLR-10262: Add support for configurable metrics implementations. (ab)
 
+* SOLR-10431: Make it possible to invoke v2 api calls using SolrJ (Cao Manh Dat, Noble Paul,
shalin)
+
 Bug Fixes
 ----------------------
 * SOLR-9262: Connection and read timeouts are being ignored by UpdateShardHandler after SOLR-4509.

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cc8b5bab/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java
index 5231dd8..157f61a 100644
--- a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java
@@ -45,6 +45,7 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.client.solrj.request.GenericSolrRequest;
 import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.request.V2Request;
 import org.apache.solr.cloud.SolrCloudTestCase;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.cloud.DocCollection;
@@ -82,10 +83,10 @@ public class BasicAuthIntegrationTest extends SolrCloudTestCase {
 
   @Test
   public void testBasicAuth() throws Exception {
-
+    boolean isUseV2Api = random().nextBoolean();
     String authcPrefix = "/admin/authentication";
     String authzPrefix = "/admin/authorization";
-    if(random().nextBoolean()){
+    if(isUseV2Api){
       authcPrefix = "/____v2/cluster/security/authentication";
       authzPrefix = "/____v2/cluster/security/authorization";
     }
@@ -110,8 +111,14 @@ public class BasicAuthIntegrationTest extends SolrCloudTestCase {
           "'set-user': {'harry':'HarryIsCool'}\n" +
           "}";
 
-      GenericSolrRequest genericReq = new GenericSolrRequest(SolrRequest.METHOD.POST, authcPrefix,
new ModifiableSolrParams());
-      genericReq.setContentStreams(Collections.singletonList(new ContentStreamBase.ByteArrayStream(command.getBytes(UTF_8),
"")));
+      final SolrRequest genericReq;
+      if (isUseV2Api) {
+        genericReq = new V2Request.Builder("/cluster/security/authentication").withMethod(SolrRequest.METHOD.POST).build();
+      } else {
+        genericReq = new GenericSolrRequest(SolrRequest.METHOD.POST, authcPrefix, new ModifiableSolrParams());
+        ((GenericSolrRequest)genericReq).setContentStreams(Collections.singletonList(new
ContentStreamBase.ByteArrayStream(command.getBytes(UTF_8), "")));
+      }
+
 
       HttpSolrClient.RemoteSolrException exp = expectThrows(HttpSolrClient.RemoteSolrException.class,
() -> {
         cluster.getSolrClient().request(genericReq);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cc8b5bab/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java
index ac388d2..3596333 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java
@@ -53,6 +53,7 @@ import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
 import org.apache.solr.client.solrj.request.IsUpdateRequest;
 import org.apache.solr.client.solrj.request.RequestWriter;
 import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.request.V2Request;
 import org.apache.solr.client.solrj.util.ClientUtils;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
@@ -1089,8 +1090,12 @@ public class CloudSolrClient extends SolrClient {
     // collections is stale and needs to be refreshed ... this code has no impact on internal
collections
     String stateVerParam = null;
     List<DocCollection> requestedCollections = null;
+    boolean isCollectionRequestOfV2 = false;
+    if (request instanceof V2Request) {
+      isCollectionRequestOfV2 = ((V2Request) request).isPerCollectionRequest();
+    }
     boolean isAdmin = ADMIN_PATHS.contains(request.getPath());
-    if (collection != null &&  !isAdmin) { // don't do _stateVer_ checking for admin
requests
+    if (collection != null &&  !isAdmin && !isCollectionRequestOfV2) { //
don't do _stateVer_ checking for admin, v2 api requests
       Set<String> requestedCollectionNames = getCollectionNames(collection);
 
       StringBuilder stateVerParamBuilder = null;
@@ -1147,8 +1152,10 @@ public class CloudSolrClient extends SolrClient {
     } catch (Exception exc) {
 
       Throwable rootCause = SolrException.getRootCause(exc);
-      // don't do retry support for admin requests or if the request doesn't have a collection
specified
-      if (collection == null || isAdmin) {
+      // don't do retry support for admin requests
+      // or if the request doesn't have a collection specified
+      // or request is v2 api and its method is not GET
+      if (collection == null || isAdmin || (request instanceof V2Request && request.getMethod()
!= SolrRequest.METHOD.GET)) {
         if (exc instanceof SolrServerException) {
           throw (SolrServerException)exc;
         } else if (exc instanceof IOException) {
@@ -1274,7 +1281,15 @@ public class CloudSolrClient extends SolrClient {
       reqParams = new ModifiableSolrParams();
     }
     List<String> theUrlList = new ArrayList<>();
-    if (ADMIN_PATHS.contains(request.getPath())) {
+    if (request instanceof V2Request) {
+      Set<String> liveNodes = stateProvider.liveNodes();
+      if (!liveNodes.isEmpty()) {
+        List<String> liveNodesList = new ArrayList<>(liveNodes);
+        Collections.shuffle(liveNodesList);
+        theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNodesList.get(0),
+            (String) stateProvider.getClusterProperty(ZkStateReader.URL_SCHEME,"http")));
+      }
+    } else if (ADMIN_PATHS.contains(request.getPath())) {
       Set<String> liveNodes = stateProvider.liveNodes();
       for (String liveNode : liveNodes) {
         theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNode,

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cc8b5bab/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java
index 5b272fc..1692aa9 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java
@@ -42,6 +42,7 @@ import org.apache.http.HttpStatus;
 import org.apache.http.NameValuePair;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpDelete;
 import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
@@ -65,6 +66,7 @@ import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.request.V2Request;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
@@ -373,6 +375,14 @@ public class HttpSolrClient extends SolrClient {
     if (collection != null)
       basePath += "/" + collection;
 
+    if (request instanceof V2Request) {
+      if (System.getProperty("solr.v2RealPath") == null) {
+        basePath = baseUrl.replace("/solr", "/v2");
+      } else {
+        basePath = baseUrl + "/____v2";
+      }
+    }
+
     if (SolrRequest.METHOD.GET == request.getMethod()) {
       if (streams != null) {
         throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!");
@@ -381,6 +391,10 @@ public class HttpSolrClient extends SolrClient {
       return new HttpGet(basePath + path + wparams.toQueryString());
     }
 
+    if (SolrRequest.METHOD.DELETE == request.getMethod()) {
+      return new HttpDelete(basePath + path + wparams.toQueryString());
+    }
+
     if (SolrRequest.METHOD.POST == request.getMethod() || SolrRequest.METHOD.PUT == request.getMethod())
{
 
       String url = basePath + path;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cc8b5bab/solr/solrj/src/java/org/apache/solr/client/solrj/request/V2Request.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/V2Request.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/V2Request.java
new file mode 100644
index 0000000..2b33593
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/V2Request.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.request;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.regex.Pattern;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.ContentStreamBase;
+
+public class V2Request extends SolrRequest {
+  static final Pattern COLL_REQ_PATTERN = Pattern.compile("/(c|collections)/[^/]+/(?!shards)");
+  private InputStream payload;
+  private SolrParams solrParams;
+
+  private V2Request(METHOD m, String resource, InputStream payload) {
+    super(m, resource);
+    this.payload = payload;
+  }
+
+  @Override
+  public SolrParams getParams() {
+    return solrParams;
+  }
+
+  @Override
+  public Collection<ContentStream> getContentStreams() throws IOException {
+    if (payload != null) {
+      return Collections.singleton(new ContentStreamBase() {
+        @Override
+        public InputStream getStream() throws IOException {
+          return payload;
+        }
+
+        @Override
+        public String getContentType() {
+          return "application/json";
+        }
+      });
+    }
+    return null;
+  }
+
+  public boolean isPerCollectionRequest() {
+    return COLL_REQ_PATTERN.matcher(getPath()).find();
+  }
+
+  @Override
+  protected SolrResponse createResponse(SolrClient client) {
+    return null;
+  }
+
+  public static class Builder {
+    private String resource;
+    private METHOD method = METHOD.GET;
+    private InputStream payload;
+    private SolrParams params;
+
+    /**
+     * Create a Builder object based on the provided resource.
+     * The default method is GET.
+     *
+     * @param resource resource of the request for example "/collections" or "/cores/core-name"
+     */
+    public Builder(String resource) {
+      if (!resource.startsWith("/")) resource = "/" + resource;
+      this.resource = resource;
+    }
+
+    public Builder withMethod(METHOD m) {
+      this.method = m;
+      return this;
+    }
+
+    /**
+     * Set payload for request.
+     * @param payload as UTF-8 String
+     * @return builder object
+     */
+    public Builder withPayload(String payload) {
+      this.payload = new ByteArrayInputStream(payload.getBytes(StandardCharsets.UTF_8));
+      return this;
+    }
+
+    public Builder withPayLoad(InputStream payload) {
+      this.payload = payload;
+      return this;
+    }
+
+    public Builder withParams(SolrParams params) {
+      this.params = params;
+      return this;
+    }
+
+    public V2Request build() {
+      V2Request v2Request = new V2Request(method, resource, payload);
+      v2Request.solrParams = params;
+      return v2Request;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cc8b5bab/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
index c3a011c..589ef7e 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
@@ -188,9 +188,7 @@ public interface CommonParams {
       CONFIGSETS_HANDLER_PATH,
       AUTHC_PATH,
       AUTHZ_PATH,
-      METRICS_PATH,
-      "/____v2/cluster/security/authentication",
-      "/____v2/cluster/security/authorization"));
+      METRICS_PATH));
 
   /** valid values for: <code>echoParams</code> */
   enum EchoParamStyle {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cc8b5bab/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV2Request.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV2Request.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV2Request.java
new file mode 100644
index 0000000..61f771e
--- /dev/null
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV2Request.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.request;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.util.NamedList;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestV2Request extends SolrCloudTestCase {
+
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    configureCluster(4)
+        .addConfig("config", getFile("solrj/solr/collection1/conf").toPath())
+        .configure();
+  }
+
+  public void assertSuccess(SolrClient client, V2Request request) throws IOException, SolrServerException
{
+    NamedList<Object> res = client.request(request);
+    assertTrue("The request failed", res.get("responseHeader").toString().contains("status=0"));
+  }
+
+  @Test
+  public void testIsCollectionRequest() {
+    assertFalse(new V2Request.Builder("/collections").build().isPerCollectionRequest());
+    assertFalse(new V2Request.Builder("/collections/a/shards").build().isPerCollectionRequest());
+    assertFalse(new V2Request.Builder("/collections/a/shards/").build().isPerCollectionRequest());
+    assertTrue(new V2Request.Builder("/collections/a/update").build().isPerCollectionRequest());
+    assertTrue(new V2Request.Builder("/c/a/update").build().isPerCollectionRequest());
+    assertTrue(new V2Request.Builder("/c/a/schema").build().isPerCollectionRequest());
+    assertFalse(new V2Request.Builder("/c/a").build().isPerCollectionRequest());
+  }
+
+  @Test
+  public void testHttpSolrClient() throws Exception {
+    HttpSolrClient solrClient = new HttpSolrClient.Builder(
+        cluster.getJettySolrRunner(0).getBaseUrl().toString()).build();
+    doTest(solrClient);
+    solrClient.close();
+  }
+
+  @Test
+  public void testCloudSolrClient() throws Exception {
+    doTest(cluster.getSolrClient());
+  }
+
+  private void doTest(SolrClient client) throws IOException, SolrServerException {
+    assertSuccess(client, new V2Request.Builder("/collections")
+        .withMethod(SolrRequest.METHOD.POST)
+        .withPayload("{" +
+            "  'create' : {" +
+            "    'name' : 'test'," +
+            "    'numShards' : 2," +
+            "    'replicationFactor' : 2," +
+            "    'config' : 'config'" +
+            "  }" +
+            "}").build());
+    assertSuccess(client, new V2Request.Builder("/c").build());
+    assertSuccess(client, new V2Request.Builder("/c/_introspect").build());
+    assertSuccess(client, new V2Request.Builder("/c/test").withMethod(SolrRequest.METHOD.DELETE).build());
+    NamedList<Object> res = client.request(new V2Request.Builder("/c").build());
+    List collections = (List) res.get("collections");
+    assertEquals(0, collections.size());
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cc8b5bab/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
index 54ab06d..cd49357 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
@@ -269,6 +269,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
     initCoreDataDir = createTempDir("init-core-data").toFile();
     System.err.println("Creating dataDir: " + initCoreDataDir.getAbsolutePath());
 
+    System.setProperty("solr.v2RealPath", "true");
     System.setProperty("zookeeper.forceSync", "no");
     System.setProperty("jetty.testMode", "true");
     System.setProperty("enable.update.log", usually() ? "true" : "false");
@@ -310,6 +311,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
       ObjectReleaseTracker.clear();
       TestInjection.reset();
       initCoreDataDir = null;
+      System.clearProperty("solr.v2RealPath");
       System.clearProperty("zookeeper.forceSync");
       System.clearProperty("jetty.testMode");
       System.clearProperty("tests.shardhandler.randomSeed");


Mime
View raw message