lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From no...@apache.org
Subject [lucene-solr] branch branch_8_4 updated: SOLR-13822: Bug fixs and tests for URP loading
Date Wed, 18 Dec 2019 22:00:04 GMT
This is an automated email from the ASF dual-hosted git repository.

noble pushed a commit to branch branch_8_4
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git


The following commit(s) were added to refs/heads/branch_8_4 by this push:
     new 221d4ed  SOLR-13822: Bug fixs and tests for URP loading
221d4ed is described below

commit 221d4ed5572ea21de792b93c228fbc0e4cb91174
Author: noble <noble@apache.org>
AuthorDate: Thu Nov 21 11:25:16 2019 +1100

    SOLR-13822: Bug fixs and tests for URP loading
---
 .../src/java/org/apache/solr/core/PluginBag.java   |   5 +--
 .../apache/solr/filestore/DistribPackageStore.java |   2 +-
 .../src/java/org/apache/solr/pkg/PackageAPI.java   |  14 +++++-
 .../processor/UpdateRequestProcessorChain.java     |   8 +++-
 .../test-files/runtimecode/TestVersionedURP.java   |  29 ++++++++++++
 solr/core/src/test-files/runtimecode/sig.txt       |  15 +++++++
 .../src/test-files/runtimecode/testurp_v1.jar.bin  | Bin 0 -> 689 bytes
 .../src/test-files/runtimecode/testurp_v2.jar.bin  | Bin 0 -> 689 bytes
 .../src/test/org/apache/solr/pkg/TestPackages.java |  50 +++++++++++++++++++--
 .../src/package-manager-internals.adoc             |  12 ++---
 10 files changed, 117 insertions(+), 18 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.java
index f547d10..b26b987 100644
--- a/solr/core/src/java/org/apache/solr/core/PluginBag.java
+++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java
@@ -47,7 +47,6 @@ import org.apache.solr.handler.component.SearchComponent;
 import org.apache.solr.pkg.PackagePluginHolder;
 import org.apache.solr.request.SolrRequestHandler;
 import org.apache.solr.update.processor.UpdateRequestProcessorChain;
-import org.apache.solr.update.processor.UpdateRequestProcessorChain.LazyUpdateProcessorFactoryHolder;
 import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
 import org.apache.solr.util.CryptoKeys;
 import org.apache.solr.util.SimplePostTool;
@@ -143,9 +142,7 @@ public class PluginBag<T> implements AutoCloseable {
     } else {
       if (info.pkgName != null) {
         PackagePluginHolder<T> holder = new PackagePluginHolder<>(info, core,
meta);
-        return meta.clazz == UpdateRequestProcessorFactory.class ?
-            new PluginHolder(info, new LazyUpdateProcessorFactoryHolder(holder)) :
-            holder;
+        return holder;
       } else {
         T inst = core.createInstance(info.className, (Class<T>) meta.clazz, meta.getCleanTag(),
null, core.getResourceLoader(info.pkgName));
         initInstance(inst, info);
diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
index 3e3b583..8d9af8f 100644
--- a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
+++ b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java
@@ -459,8 +459,8 @@ public class DistribPackageStore implements PackageStore {
       if (fetch(path, null)) {
         file = getRealpath(path).toFile();
       }
-      if (!file.exists()) return FileType.NOFILE;
     }
+    if (!file.exists()) return FileType.NOFILE;
     if (file.isDirectory()) return FileType.DIRECTORY;
     return isMetaDataFile(file.getName()) ? FileType.METADATA : FileType.FILE;
   }
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
index d80e8bd..1b93d89 100644
--- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
@@ -271,8 +271,18 @@ public class PackageAPI {
             log.error("Error deserializing packages.json", e);
             packages = new Packages();
           }
-          packages.packages.computeIfAbsent(add.pkg, Utils.NEW_ARRAYLIST_FUN).add(new PkgVersion(add));
-          packages.znodeVersion = stat.getVersion() ;
+          List list = packages.packages.computeIfAbsent(add.pkg, Utils.NEW_ARRAYLIST_FUN);
+          for (Object o : list) {
+            if (o instanceof PkgVersion) {
+              PkgVersion version = (PkgVersion) o;
+              if (Objects.equals(version.version, add.version)) {
+                payload.addError("Version '" + add.version + "' exists already");
+                return null;
+              }
+            }
+          }
+          list.add(new PkgVersion(add));
+          packages.znodeVersion = stat.getVersion() + 1;
           finalState[0] = packages;
           return Utils.toJSON(packages);
         });
diff --git a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java
b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java
index 5ddaf8b..a912e2a 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java
@@ -296,7 +296,13 @@ public final class UpdateRequestProcessorChain implements PluginInfoInitialized
     for (String s : names) {
       s = s.trim();
       if (s.isEmpty()) continue;
-      UpdateRequestProcessorFactory p = core.getUpdateProcessors().get(s);
+      UpdateRequestProcessorFactory p = null;
+      PluginBag.PluginHolder<UpdateRequestProcessorFactory> holder = core.getUpdateProcessors().getRegistry().get(s);
+      if (holder instanceof PackagePluginHolder) {
+        p = new LazyUpdateRequestProcessorFactory(holder);
+      } else {
+        p = core.getUpdateProcessors().get(s);
+      }
       if (p == null) {
         Class<UpdateRequestProcessorFactory> factoryClass = implicits.get(s);
         if(factoryClass != null) {
diff --git a/solr/core/src/test-files/runtimecode/TestVersionedURP.java b/solr/core/src/test-files/runtimecode/TestVersionedURP.java
new file mode 100644
index 0000000..ae91b80
--- /dev/null
+++ b/solr/core/src/test-files/runtimecode/TestVersionedURP.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package runtimecode;
+
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.processor.SimpleUpdateProcessorFactory;
+
+public class TestVersionedURP extends SimpleUpdateProcessorFactory {
+  @Override
+  protected void process(AddUpdateCommand cmd, SolrQueryRequest req, SolrQueryResponse rsp)
{
+    cmd.solrDoc.addField("TestVersionedURP.Ver_s", "Version 2");
+  }
+}
diff --git a/solr/core/src/test-files/runtimecode/sig.txt b/solr/core/src/test-files/runtimecode/sig.txt
index 4ef8e9c..f354b5f 100644
--- a/solr/core/src/test-files/runtimecode/sig.txt
+++ b/solr/core/src/test-files/runtimecode/sig.txt
@@ -57,6 +57,14 @@ openssl dgst -sha1 -sign ../cryptokeys/priv_key512.pem cache_v2.jar.bin
| openss
 SOrekHt+uup+z2z+nZU5indk2huRRfmbM+W+vQ0variHrcZEG9EXt5LuPFl8Ki9A
 hr6klMHdVP8nj4wuQhu/Hg==
 
+openssl dgst -sha1 -sign ../cryptokeys/priv_key512.pem testurp_v1.jar.bin | openssl enc -base64
| tr -d \\n | sed
+
+h6UmMzuPqu4hQFGLBMJh/6kDSEXpJlgLsQDXx0KuxXWkV5giilRP57K3towiJRh2J+rqihqIghNCi3YgzgUnWQ==
+
+openssl dgst -sha1 -sign ../cryptokeys/priv_key512.pem testurp_v2.jar.bin | openssl enc -base64
| openssl enc -base64 | tr -d \\n | sed
+
+P/ptFXRvQMd4oKPvadSpd+A9ffwY3gcex5GVFVRy3df0/OF8XT5my8rQz7FZva+2ORbWxdXS8NKwNrbPVHLGXw==
+
 ====================sha512====================
 
 openssl dgst -sha512 runtimelibs.jar.bin
@@ -80,6 +88,13 @@ openssl dgst -sha512 cache_v2.jar.bin
 
 873337e67b90b7ff99df012b2e9093c63079c37a564643d34861a88c4cbaf0698ebb096905929d69cdbde3b4d29d55e31db24ee05c01b39c0b75a16e54eb4335
 
+openssl dgst -sha512 testurp_v1.jar.bin
+
+7b9df184202e62cc6f73e69a7f369a6f469ad9e1508a2d61eeb7744b204f2489a2c617808f28a496fcbca3a318e16dca238c111e60a903ace877a79900bb8729
+
+openssl dgst -sha512 testurp_v2.jar.bin
+
+5c4c0c454a032916e48a1c14a0fecbd6658658a66aedec5168b7222f2e3c0c63fbe09637238a9325ce2e95a2c8521834397a97701ead46c681aa20c9fccb6654
 =============sha256============================
 
 openssl dgst -sha256 runtimelibs.jar.bin
diff --git a/solr/core/src/test-files/runtimecode/testurp_v1.jar.bin b/solr/core/src/test-files/runtimecode/testurp_v1.jar.bin
new file mode 100644
index 0000000..7c9044c
Binary files /dev/null and b/solr/core/src/test-files/runtimecode/testurp_v1.jar.bin differ
diff --git a/solr/core/src/test-files/runtimecode/testurp_v2.jar.bin b/solr/core/src/test-files/runtimecode/testurp_v2.jar.bin
new file mode 100644
index 0000000..0c8041e
Binary files /dev/null and b/solr/core/src/test-files/runtimecode/testurp_v2.jar.bin differ
diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
index 560cdea..baa8a99 100644
--- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
+++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
@@ -26,6 +26,7 @@ import java.util.concurrent.Callable;
 
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -34,14 +35,17 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.client.solrj.request.GenericSolrRequest;
 import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.request.UpdateRequest;
 import org.apache.solr.client.solrj.request.V2Request;
 import org.apache.solr.client.solrj.request.beans.Package;
+import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.client.solrj.util.ClientUtils;
 import org.apache.solr.cloud.ConfigRequest;
 import org.apache.solr.cloud.MiniSolrCloudCluster;
 import org.apache.solr.cloud.SolrCloudTestCase;
 import org.apache.solr.common.MapWriterMap;
 import org.apache.solr.common.NavigableObject;
+import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.params.MapSolrParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.params.SolrParams;
@@ -86,6 +90,8 @@ public class TestPackages extends SolrCloudTestCase {
       String FILE1 = "/mypkg/runtimelibs.jar";
       String FILE2 = "/mypkg/runtimelibs_v2.jar";
       String FILE3 = "/mypkg/runtimelibs_v3.jar";
+      String URP1 = "/mypkg/testurpv1.jar";
+      String URP2 = "/mypkg/testurpv2.jar";
       String COLLECTION_NAME = "testPluginLoadingColl";
       byte[] derFile = readFile("cryptokeys/pub_key512.der");
       cluster.getZkClient().makePath("/keys/exe", true);
@@ -93,10 +99,13 @@ public class TestPackages extends SolrCloudTestCase {
       postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1,
           "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ==");
 
+      postFileAndWait(cluster, "runtimecode/testurp_v1.jar.bin", URP1,
+          "h6UmMzuPqu4hQFGLBMJh/6kDSEXpJlgLsQDXx0KuxXWkV5giilRP57K3towiJRh2J+rqihqIghNCi3YgzgUnWQ==");
+
       Package.AddVersion add = new Package.AddVersion();
       add.version = "1.0";
       add.pkg = "mypkg";
-      add.files = Arrays.asList(new String[]{FILE1});
+      add.files = Arrays.asList(new String[]{FILE1, URP1});
       V2Request req = new V2Request.Builder("/cluster/package")
           .forceV2(true)
           .withMethod(SolrRequest.METHOD.POST)
@@ -125,6 +134,7 @@ public class TestPackages extends SolrCloudTestCase {
           "'create-requesthandler' : { 'name' : '/runtime', 'class': 'mypkg:org.apache.solr.core.RuntimeLibReqHandler'
}," +
           "'create-searchcomponent' : { 'name' : 'get', 'class': 'mypkg:org.apache.solr.core.RuntimeLibSearchComponent'
 }," +
           "'create-queryResponseWriter' : { 'name' : 'json1', 'class': 'mypkg:org.apache.solr.core.RuntimeLibResponseWriter'
}" +
+          "'create-updateProcessor' : { 'name' : 'myurp', 'class': 'mypkg:org.apache.solr.update.TestVersionedURP'
}" +
           "}";
       cluster.getSolrClient().request(new ConfigRequest(payload) {
         @Override
@@ -145,7 +155,20 @@ public class TestPackages extends SolrCloudTestCase {
           COLLECTION_NAME, "requestHandler", "/runtime",
           "mypkg", "1.0" );
 
+      verifyCmponent(cluster.getSolrClient(),
+          COLLECTION_NAME, "updateProcessor", "myurp",
+          "mypkg", "1.0" );
+
+      UpdateRequest ur = new UpdateRequest();
+      ur.add(new SolrInputDocument("id", "1"));
+      ur.setParam("processor", "myurp");
+      ur.process(cluster.getSolrClient(), COLLECTION_NAME);
+      cluster.getSolrClient().commit(COLLECTION_NAME, true, true);
+
+      QueryResponse result = cluster.getSolrClient()
+          .query(COLLECTION_NAME, new SolrQuery( "id:1"));
 
+      assertEquals("Version 1", result.getResults().get(0).getFieldValue("TestVersionedURP.Ver_s"));
 
       executeReq( "/" + COLLECTION_NAME + "/runtime?wt=javabin", cluster.getRandomJetty(random()),
           Utils.JAVABINCONSUMER,
@@ -165,9 +188,11 @@ public class TestPackages extends SolrCloudTestCase {
       postFileAndWait(cluster, "runtimecode/runtimelibs_v2.jar.bin", FILE2,
           "j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA==");
 
+      postFileAndWait(cluster, "runtimecode/testurp_v2.jar.bin", URP2,
+          "P/ptFXRvQMd4oKPvadSpd+A9ffwY3gcex5GVFVRy3df0/OF8XT5my8rQz7FZva+2ORbWxdXS8NKwNrbPVHLGXw==");
       //add the version using package API
       add.version = "1.1";
-      add.files = Arrays.asList(new String[]{FILE2});
+      add.files = Arrays.asList(new String[]{FILE2,URP2});
       req.process(cluster.getSolrClient());
 
       verifyCmponent(cluster.getSolrClient(),
@@ -182,6 +207,11 @@ public class TestPackages extends SolrCloudTestCase {
           COLLECTION_NAME, "requestHandler", "/runtime",
           "mypkg", "1.1" );
 
+      verifyCmponent(cluster.getSolrClient(),
+          COLLECTION_NAME, "updateProcessor", "myurp",
+          "mypkg", "1.1" );
+
+
       executeReq( "/" + COLLECTION_NAME + "/get?wt=json", cluster.getRandomJetty(random()),
           Utils.JSONCONSUMER,
           Utils.makeMap(  "Version","2"));
@@ -192,7 +222,7 @@ public class TestPackages extends SolrCloudTestCase {
           "a400n4T7FT+2gM0SC6+MfSOExjud8MkhTSFylhvwNjtWwUgKdPFn434Wv7Qc4QEqDVLhQoL3WqYtQmLPti0G4Q==");
 
       add.version = "2.1";
-      add.files = Arrays.asList(new String[]{FILE3});
+      add.files = Arrays.asList(new String[]{FILE3, URP2});
       req.process(cluster.getSolrClient());
 
       //now let's verify that the classes are updated
@@ -212,6 +242,18 @@ public class TestPackages extends SolrCloudTestCase {
           Utils.JSONCONSUMER,
           Utils.makeMap("Version","2"));
 
+      //insert a doc with urp
+      ur = new UpdateRequest();
+      ur.add(new SolrInputDocument("id", "2"));
+      ur.setParam("processor", "myurp");
+      ur.process(cluster.getSolrClient(), COLLECTION_NAME);
+      cluster.getSolrClient().commit(COLLECTION_NAME, true, true);
+
+      result = cluster.getSolrClient()
+          .query(COLLECTION_NAME, new SolrQuery( "id:2"));
+
+      assertEquals("Version 2", result.getResults().get(0).getFieldValue("TestVersionedURP.Ver_s"));
+
 
       Package.DelVersion delVersion = new Package.DelVersion();
       delVersion.pkg = "mypkg";
@@ -262,7 +304,7 @@ public class TestPackages extends SolrCloudTestCase {
       }.process(cluster.getSolrClient()) ;
 
       add.version = "2.1";
-      add.files = Arrays.asList(new String[]{FILE3});
+      add.files = Arrays.asList(new String[]{FILE3, URP2});
       req.process(cluster.getSolrClient());
 
       //the collections mypkg is set to use version 1.1
diff --git a/solr/solr-ref-guide/src/package-manager-internals.adoc b/solr/solr-ref-guide/src/package-manager-internals.adoc
index 90ed865..6cb1956 100644
--- a/solr/solr-ref-guide/src/package-manager-internals.adoc
+++ b/solr/solr-ref-guide/src/package-manager-internals.adoc
@@ -1,7 +1,7 @@
 = Package Manager Internals
 The package manager (CLI) internally uses various Solr APIs to install, deploy and update
packages. This document contains an overview of those APIs.
 
-== Design Objectives
+== Salient Features
 
 * Zero disruption deployment (hot deployment): Should be possible to install and update packages
without node restarts or core reloads, and hence deployments should be quick and without failed
requests or stale caches.
 * Easy packaging:
@@ -81,13 +81,13 @@ $ curl -o runtimelibs.jar   -LO https://github.com/apache/lucene-solr/blob/maste
 2) Sign the jar with your private key
 [source, bash]
 ----
-$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64
+$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64 | sed 's/+/%2B/g'
| tr -d \\n | sed
 ----
 
-3) Upload your jar with signature. (replace the `sig` param with the output you got from
the previous command) . Ensure that new lines and spaces are removed. Do not forget to do
URL encoding of you signature( e.g   escape `+` with `%2B`)
+3) Upload your jar with signature. (replace the `sig` param with the output you got from
the previous command)
 [source, bash]
 ----
-$ curl --data-binary @runtimelibs.jar -X PUT  http://localhost:7574/api/cluster/files/mypkg/1.0/myplugins.jar?sig=elNjhmWIOgTgbAzeZ%2BOcwR42N7vqL6Ig9eAqn4YoP2thT7FJuhiaZuCPivjMkD682EBo9gveSCTyXIsZKjOCbQ==
+$ curl --data-binary @runtimelibs.jar -X PUT  http://localhost:7574/api/cluster/files/mypkg/1.0/myplugins.jar?sig=<signature-of-jar>
 ----
 
 4) Verify your jar upload
@@ -263,8 +263,8 @@ Get a new version of the jar, sign and upload it
 [source, bash]
 ----
 $ curl -o runtimelibs3.jar   -LO https://github.com/apache/lucene-solr/blob/master/solr/core/src/test-files/runtimecode/runtimelibs_v3.jar.bin?raw=true
-$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64
-$ curl --data-binary @runtimelibs3.jar -X PUT  http://localhost:8983/api/cluster/files/mypkg/2.0/myplugins.jar?sig=ICkC%2BnGE%2BAqiANM0ajhVPNCQsbPbHLSWlIe5ETV5835e5HqndWrFHiV2R6nLVjDCxov/wLPo1uK0VzvAPIioUQ==
+$ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64 | sed 's/+/%2B/g'
| tr -d \\n | sed
+$ curl --data-binary @runtimelibs3.jar -X PUT  http://localhost:8983/api/cluster/files/mypkg/2.0/myplugins.jar?sig=<signature>
 ----
 
 7) Verify it


Mime
View raw message