lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From is...@apache.org
Subject [lucene-solr] 01/10: SOLR-13822: Isolated class loading from packages
Date Mon, 28 Oct 2019 22:58:43 GMT
This is an automated email from the ASF dual-hosted git repository.

ishan pushed a commit to branch jira/solr-13662-2
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git

commit fe77a4596d8f5b02d7dadf1f11c211a4e8b76efa
Author: noble <noble@apache.org>
AuthorDate: Sat Oct 12 17:16:43 2019 +1100

    SOLR-13822: Isolated class loading from packages
---
 .../java/org/apache/solr/core/CoreContainer.java   |  10 +
 .../src/java/org/apache/solr/core/PluginBag.java   |  15 +-
 .../src/java/org/apache/solr/core/PluginInfo.java  |  39 +-
 .../src/java/org/apache/solr/core/SolrCore.java    |  18 +-
 .../org/apache/solr/core/SolrResourceLoader.java   | 333 ++++++++++--------
 .../org/apache/solr/handler/SolrConfigHandler.java |  23 +-
 .../src/java/org/apache/solr/pkg/PackageAPI.java   | 367 +++++++++++++++++++
 .../java/org/apache/solr/pkg/PackageListeners.java |  89 +++++
 .../java/org/apache/solr/pkg/PackageLoader.java    | 254 +++++++++++++
 .../org/apache/solr/pkg/PackagePluginHolder.java   |  92 +++++
 .../solr/security/PermissionNameProvider.java      |   3 +
 .../src/test/org/apache/solr/pkg/TestPackages.java | 391 +++++++++++++++++++++
 .../org/apache/solr/common/cloud/SolrZkClient.java |   9 +-
 .../apache/solr/common/cloud/ZkStateReader.java    |   1 +
 14 files changed, 1471 insertions(+), 173 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 054bd67..c32783c 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -100,6 +100,7 @@ import org.apache.solr.logging.MDCLoggingContext;
 import org.apache.solr.metrics.SolrCoreMetricManager;
 import org.apache.solr.metrics.SolrMetricManager;
 import org.apache.solr.metrics.SolrMetricProducer;
+import org.apache.solr.pkg.PackageLoader;
 import org.apache.solr.request.SolrRequestHandler;
 import org.apache.solr.request.SolrRequestInfo;
 import org.apache.solr.search.SolrFieldCacheBean;
@@ -221,6 +222,7 @@ public class CoreContainer {
   protected volatile AutoscalingHistoryHandler autoscalingHistoryHandler;
 
   private PackageStoreAPI packageStoreAPI;
+  private PackageLoader packageLoader;
 
 
   // Bits for the state variable.
@@ -580,6 +582,10 @@ public class CoreContainer {
     return replayUpdatesExecutor;
   }
 
+  public PackageLoader getPackageLoader() {
+    return packageLoader;
+  }
+
   public PackageStoreAPI getPackageStoreAPI() {
     return packageStoreAPI;
   }
@@ -734,6 +740,10 @@ public class CoreContainer {
     if (isZooKeeperAware()) {
       metricManager.loadClusterReporters(metricReporters, this);
     }
+    packageLoader = new PackageLoader(this);
+    containerHandlers.getApiBag().register(new AnnotatedApi(packageLoader.getPackageAPI().editAPI), Collections.EMPTY_MAP);
+    containerHandlers.getApiBag().register(new AnnotatedApi(packageLoader.getPackageAPI().readAPI), Collections.EMPTY_MAP);
+
 
     // setup executor to load cores in parallel
     ExecutorService coreLoadExecutor = MetricUtils.instrumentedExecutorService(
diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.java
index 6088f52..bfdfa20 100644
--- a/solr/core/src/java/org/apache/solr/core/PluginBag.java
+++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java
@@ -43,6 +43,7 @@ import org.apache.solr.common.SolrException;
 import org.apache.solr.common.util.StrUtils;
 import org.apache.solr.handler.RequestHandlerBase;
 import org.apache.solr.handler.component.SearchComponent;
+import org.apache.solr.pkg.PackagePluginHolder;
 import org.apache.solr.request.SolrRequestHandler;
 import org.apache.solr.update.processor.UpdateRequestProcessorChain;
 import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
@@ -97,7 +98,7 @@ public class PluginBag<T> implements AutoCloseable {
     this(klass, core, false);
   }
 
-  static void initInstance(Object inst, PluginInfo info) {
+  public static void initInstance(Object inst, PluginInfo info) {
     if (inst instanceof PluginInfoInitialized) {
       ((PluginInfoInitialized) inst).init(info);
     } else if (inst instanceof NamedListInitializedPlugin) {
@@ -138,9 +139,13 @@ public class PluginBag<T> implements AutoCloseable {
       log.debug("{} : '{}' created with startup=lazy ", meta.getCleanTag(), info.name);
       return new LazyPluginHolder<T>(meta, info, core, core.getResourceLoader(), false);
     } else {
-      T inst = core.createInstance(info.className, (Class<T>) meta.clazz, meta.getCleanTag(), null, core.getResourceLoader());
-      initInstance(inst, info);
-      return new PluginHolder<>(info, inst);
+      if (info.pkgName != null) {
+        return new PackagePluginHolder<>(info, core, meta);
+      } else {
+        T inst = core.createInstance(info.className, (Class<T>) meta.clazz, meta.getCleanTag(), null, core.getResourceLoader(info.pkgName));
+        initInstance(inst, info);
+        return new PluginHolder<>(info, inst);
+      }
     }
   }
 
@@ -329,7 +334,7 @@ public class PluginBag<T> implements AutoCloseable {
    * subclasses may choose to lazily load the plugin
    */
   public static class PluginHolder<T> implements AutoCloseable {
-    private T inst;
+    protected T inst;
     protected final PluginInfo pluginInfo;
     boolean registerAPI = false;
 
diff --git a/solr/core/src/java/org/apache/solr/core/PluginInfo.java b/solr/core/src/java/org/apache/solr/core/PluginInfo.java
index 1bc85ae..366309e 100644
--- a/solr/core/src/java/org/apache/solr/core/PluginInfo.java
+++ b/solr/core/src/java/org/apache/solr/core/PluginInfo.java
@@ -16,14 +16,20 @@
  */
 package org.apache.solr.core;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
 import org.apache.solr.common.MapSerializable;
 import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.Pair;
 import org.apache.solr.util.DOMUtil;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 
-import java.util.*;
-
 import static java.util.Arrays.asList;
 import static java.util.Collections.unmodifiableList;
 import static java.util.Collections.unmodifiableMap;
@@ -35,27 +41,46 @@ import static org.apache.solr.schema.FieldType.CLASS_NAME;
  *
  */
 public class PluginInfo implements MapSerializable {
-  public final String name, className, type;
+  public final String name, className, type, pkgName;
   public final NamedList initArgs;
   public final Map<String, String> attributes;
   public final List<PluginInfo> children;
   private boolean isFromSolrConfig;
 
+
+
   public PluginInfo(String type, Map<String, String> attrs, NamedList initArgs, List<PluginInfo> children) {
     this.type = type;
     this.name = attrs.get(NAME);
-    this.className = attrs.get(CLASS_NAME);
+    Pair<String, String> parsed = parseClassName(attrs.get(CLASS_NAME));
+    this.className = parsed.second();
+    this.pkgName = parsed.first();
     this.initArgs = initArgs;
     attributes = unmodifiableMap(attrs);
     this.children = children == null ? Collections.<PluginInfo>emptyList(): unmodifiableList(children);
     isFromSolrConfig = false;
   }
+  static Pair<String,String > parseClassName(String name) {
+    String pkgName = null;
+    String className = name;
+    if (name != null) {
+      int colonIdx = name.indexOf(':');
+      if (colonIdx > -1) {
+        pkgName = name.substring(0, colonIdx);
+        className = name.substring(colonIdx + 1);
+      }
+    }
+    return new Pair<>(pkgName, className);
+
+  }
 
 
   public PluginInfo(Node node, String err, boolean requireName, boolean requireClass) {
     type = node.getNodeName();
     name = DOMUtil.getAttr(node, NAME, requireName ? err : null);
-    className = DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null);
+    Pair<String, String> parsed = parseClassName(DOMUtil.getAttr(node, CLASS_NAME, requireClass ? err : null));
+    className = parsed.second();
+    pkgName = parsed.first();
     initArgs = DOMUtil.childNodesToNamedList(node);
     attributes = unmodifiableMap(DOMUtil.toMap(node.getAttributes()));
     children = loadSubPlugins(node);
@@ -85,7 +110,9 @@ public class PluginInfo implements MapSerializable {
     }
     this.type = type;
     this.name = (String) m.get(NAME);
-    this.className = (String) m.get(CLASS_NAME);
+    Pair<String, String> parsed = parseClassName((String) m.get(CLASS_NAME));
+    this.className = parsed.second();
+    this.pkgName = parsed.first();
     attributes = unmodifiableMap(m);
     this.children =  Collections.<PluginInfo>emptyList();
     isFromSolrConfig = true;
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 3e2fb1e..59c9a7a 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -110,6 +110,8 @@ import org.apache.solr.logging.MDCLoggingContext;
 import org.apache.solr.metrics.SolrCoreMetricManager;
 import org.apache.solr.metrics.SolrMetricManager;
 import org.apache.solr.metrics.SolrMetricProducer;
+import org.apache.solr.pkg.PackageListeners;
+import org.apache.solr.pkg.PackageLoader;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.request.SolrRequestHandler;
 import org.apache.solr.response.BinaryResponseWriter;
@@ -237,6 +239,8 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
   public volatile boolean indexEnabled = true;
   public volatile boolean readOnly = false;
 
+  private PackageListeners packageListeners = new PackageListeners();
+
   public Set<String> getMetricNames() {
     return metricNames;
   }
@@ -261,6 +265,10 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
     return restManager;
   }
 
+  public PackageListeners getPackageListeners() {
+    return packageListeners;
+  }
+
   static int boolean_query_max_clause_count = Integer.MIN_VALUE;
 
   private ExecutorService coreAsyncTaskExecutor = ExecutorUtil.newMDCAwareCachedThreadPool("Core Async Task");
@@ -274,6 +282,14 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
     return resourceLoader;
   }
 
+  public SolrResourceLoader getResourceLoader(String pkg) {
+    if (pkg == null) {
+      return resourceLoader;
+    }
+    PackageLoader.Package aPackage = coreContainer.getPackageLoader().getPackage(pkg);
+    return aPackage.getLatest().getLoader();
+  }
+
   /**
    * Gets the configuration resource name used by this core instance.
    *
@@ -856,7 +872,7 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
 
   public <T extends Object> T createInitInstance(PluginInfo info, Class<T> cast, String msg, String defClassName) {
     if (info == null) return null;
-    T o = createInstance(info.className == null ? defClassName : info.className, cast, msg, this, getResourceLoader());
+    T o = createInstance(info.className == null ? defClassName : info.className, cast, msg, this, getResourceLoader(info.pkgName));
     if (o instanceof PluginInfoInitialized) {
       ((PluginInfoInitialized) o).init(info);
     } else if (o instanceof NamedListInitializedPlugin) {
diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
index 4132918..cc1ef7a 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
@@ -28,6 +28,7 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.lang.invoke.MethodHandles;
 import java.lang.reflect.Constructor;
+import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.nio.charset.CharacterCodingException;
@@ -81,11 +82,11 @@ import org.slf4j.LoggerFactory;
 
 /**
  * @since solr 1.3
- */ 
-public class SolrResourceLoader implements ResourceLoader,Closeable
-{
+ */
+public class SolrResourceLoader implements ResourceLoader, Closeable {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
+  private String name = "";
   static final String project = "solr";
   static final String base = "org.apache" + "." + project;
   static final String[] packages = {
@@ -100,7 +101,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
   protected URLClassLoader classLoader;
   private final Path instanceDir;
   private String dataDir;
-  
+
   private final List<SolrCoreAware> waitingForCore = Collections.synchronizedList(new ArrayList<SolrCoreAware>());
   private final List<SolrInfoBean> infoMBeans = Collections.synchronizedList(new ArrayList<SolrInfoBean>());
   private final List<ResourceLoaderAware> waitingForResources = Collections.synchronizedList(new ArrayList<ResourceLoaderAware>());
@@ -109,18 +110,18 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
   private final Properties coreProperties;
 
   private volatile boolean live;
-  
+
   // Provide a registry so that managed resources can register themselves while the XML configuration
   // documents are being parsed ... after all are registered, they are asked by the RestManager to
   // initialize themselves. This two-step process is required because not all resources are available
   // (such as the SolrZkClient) when XML docs are being parsed.    
   private RestManager.Registry managedResourceRegistry;
-  
+
   public synchronized RestManager.Registry getManagedResourceRegistry() {
     if (managedResourceRegistry == null) {
-      managedResourceRegistry = new RestManager.Registry();      
+      managedResourceRegistry = new RestManager.Registry();
     }
-    return managedResourceRegistry; 
+    return managedResourceRegistry;
   }
 
   public SolrResourceLoader() {
@@ -134,11 +135,20 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
    * found in the "lib/" directory in the specified instance directory.
    * If the instance directory is not specified (=null), SolrResourceLoader#locateInstanceDir will provide one.
    */
-  public SolrResourceLoader(Path instanceDir, ClassLoader parent)
-  {
+  public SolrResourceLoader(Path instanceDir, ClassLoader parent) {
     this(instanceDir, parent, null);
   }
 
+  public SolrResourceLoader(String name, List<Path> classpath, Path instanceDir, ClassLoader parent) throws MalformedURLException {
+    this(instanceDir, parent);
+    this.name = name;
+    for (Path path : classpath) {
+      addToClassLoader(path.toUri().normalize().toURL());
+    }
+
+  }
+
+
   public SolrResourceLoader(Path instanceDir) {
     this(instanceDir, null, null);
   }
@@ -157,7 +167,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
     if (instanceDir == null) {
       this.instanceDir = SolrResourceLoader.locateSolrHome().toAbsolutePath().normalize();
       log.debug("new SolrResourceLoader for deduced Solr Home: '{}'", this.instanceDir);
-    } else{
+    } else {
       this.instanceDir = instanceDir.toAbsolutePath().normalize();
       log.debug("new SolrResourceLoader for directory: '{}'", this.instanceDir);
     }
@@ -167,7 +177,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
     }
     this.classLoader = URLClassLoader.newInstance(new URL[0], parent);
 
-    /* 
+    /*
      * Skip the lib subdirectory when we are loading from the solr home.
      * Otherwise load it, so core lib directories still get loaded.
      * The default sharedLib will pick this up later, and if the user has
@@ -264,6 +274,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
 
   /**
    * Utility method to get the URLs of all paths under a given directory that match a filter
+   *
    * @param libDir the root directory
    * @param filter the filter
    * @return all matching URLs
@@ -296,8 +307,9 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
 
   /**
    * Utility method to get the URLs of all paths under a given directory that match a regex
+   *
    * @param libDir the root directory
-   * @param regex the regex as a String
+   * @param regex  the regex as a String
    * @return all matching URLs
    * @throws IOException on error
    */
@@ -310,15 +322,17 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
       }
     });
   }
-  
-  /** Ensures a directory name always ends with a '/'. */
+
+  /**
+   * Ensures a directory name always ends with a '/'.
+   */
   public static String normalizeDir(String path) {
-    return ( path != null && (!(path.endsWith("/") || path.endsWith("\\"))) )? path + File.separator : path;
+    return (path != null && (!(path.endsWith("/") || path.endsWith("\\")))) ? path + File.separator : path;
   }
-  
+
   public String[] listConfigDir() {
     File configdir = new File(getConfigDir());
-    if( configdir.exists() && configdir.isDirectory() ) {
+    if (configdir.exists() && configdir.isDirectory()) {
       return configdir.list();
     } else {
       return new String[0];
@@ -328,8 +342,8 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
   public String getConfigDir() {
     return instanceDir.resolve("conf").toString();
   }
-  
-  public String getDataDir()    {
+
+  public String getDataDir() {
     return dataDir;
   }
 
@@ -341,23 +355,28 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
    * EXPERT
    * <p>
    * The underlying class loader.  Most applications will not need to use this.
+   *
    * @return The {@link ClassLoader}
    */
   public ClassLoader getClassLoader() {
     return classLoader;
   }
 
-  /** Opens a schema resource by its name.
+  /**
+   * Opens a schema resource by its name.
    * Override this method to customize loading schema resources.
-   *@return the stream for the named schema
+   *
+   * @return the stream for the named schema
    */
   public InputStream openSchema(String name) throws IOException {
     return openResource(name);
   }
-  
-  /** Opens a config resource by its name.
+
+  /**
+   * Opens a config resource by its name.
    * Override this method to customize loading config resources.
-   *@return the stream for the named configuration
+   *
+   * @return the stream for the named configuration
    */
   public InputStream openConfig(String name) throws IOException {
     return openResource(name);
@@ -372,14 +391,16 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
     throw new IOException("File " + pathToCheck + " is outside resource loader dir " + instanceDir +
         "; set -Dsolr.allow.unsafe.resourceloading=true to allow unsafe loading");
   }
-  
-  /** Opens any resource by its name.
+
+  /**
+   * Opens any resource by its name.
    * By default, this will look in multiple locations to load the resource:
    * $configDir/$resource (if resource is not absolute)
    * $CWD/$resource
    * otherwise, it will look for it in any jar accessible through the class loader.
    * Override this method to customize loading resources.
-   *@return the stream for the named resource
+   *
+   * @return the stream for the named resource
    */
   @Override
   public InputStream openResource(String resource) throws IOException {
@@ -461,22 +482,22 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
    * @throws IOException If there is a low-level I/O error.
    */
   public List<String> getLines(String resource,
-      String encoding) throws IOException {
+                               String encoding) throws IOException {
     return getLines(resource, Charset.forName(encoding));
   }
 
 
-  public List<String> getLines(String resource, Charset charset) throws IOException{
+  public List<String> getLines(String resource, Charset charset) throws IOException {
     try {
       return WordlistLoader.getLines(openResource(resource), charset);
     } catch (CharacterCodingException ex) {
-      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, 
-         "Error loading resource (wrong encoding?): " + resource, ex);
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+          "Error loading resource (wrong encoding?): " + resource, ex);
     }
   }
 
   /*
-   * A static map of short class name to fully qualified class name 
+   * A static map of short class name to fully qualified class name
    */
   private static final Map<String, String> classNameCache = new ConcurrentHashMap<>();
 
@@ -486,14 +507,14 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
   }
 
   // Using this pattern, legacy analysis components from previous Solr versions are identified and delegated to SPI loader:
-  private static final Pattern legacyAnalysisPattern = 
-      Pattern.compile("((\\Q"+base+".analysis.\\E)|(\\Q"+project+".\\E))([\\p{L}_$][\\p{L}\\p{N}_$]+?)(TokenFilter|Filter|Tokenizer|CharFilter)Factory");
+  private static final Pattern legacyAnalysisPattern =
+      Pattern.compile("((\\Q" + base + ".analysis.\\E)|(\\Q" + project + ".\\E))([\\p{L}_$][\\p{L}\\p{N}_$]+?)(TokenFilter|Filter|Tokenizer|CharFilter)Factory");
 
   @Override
   public <T> Class<? extends T> findClass(String cname, Class<T> expectedType) {
     return findClass(cname, expectedType, empty);
   }
-  
+
   /**
    * This method loads a class either with its FQN or a short-name (solr.class-simplename or class-simplename).
    * It tries to load the class with the name that is given first and if it fails, it tries all the known
@@ -501,25 +522,25 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
    * for the same class faster. The caching is done only if the class is loaded by the webapp classloader and it
    * is loaded using a shortname.
    *
-   * @param cname The name or the short name of the class.
+   * @param cname       The name or the short name of the class.
    * @param subpackages the packages to be tried if the cname starts with solr.
    * @return the loaded class. An exception is thrown if it fails
    */
   public <T> Class<? extends T> findClass(String cname, Class<T> expectedType, String... subpackages) {
     if (subpackages == null || subpackages.length == 0 || subpackages == packages) {
       subpackages = packages;
-      String  c = classNameCache.get(cname);
-      if(c != null) {
+      String c = classNameCache.get(cname);
+      if (c != null) {
         try {
           return Class.forName(c, true, classLoader).asSubclass(expectedType);
         } catch (ClassNotFoundException | ClassCastException e) {
           // this can happen if the legacyAnalysisPattern below caches the wrong thing
-          log.warn("Unable to load cached class, attempting lookup. name={} shortname={} reason={}", c, cname, e);
+          log.warn( name + " Unable to load cached class, attempting lookup. name={} shortname={} reason={}", c, cname, e);
           classNameCache.remove(cname);
         }
       }
     }
-    
+
     Class<? extends T> clazz = null;
     try {
       // first try legacy analysis patterns, now replaced by Lucene's Analysis package:
@@ -537,43 +558,43 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
           } else {
             log.warn("'{}' looks like an analysis factory, but caller requested different class type: {}", cname, expectedType.getName());
           }
-        } catch (IllegalArgumentException ex) { 
+        } catch (IllegalArgumentException ex) {
           // ok, we fall back to legacy loading
         }
       }
-      
+
       // first try cname == full name
       try {
         return clazz = Class.forName(cname, true, classLoader).asSubclass(expectedType);
       } catch (ClassNotFoundException e) {
-        String newName=cname;
+        String newName = cname;
         if (newName.startsWith(project)) {
-          newName = cname.substring(project.length()+1);
+          newName = cname.substring(project.length() + 1);
         }
         for (String subpackage : subpackages) {
           try {
             String name = base + '.' + subpackage + newName;
             log.trace("Trying class name " + name);
-            return clazz = Class.forName(name,true,classLoader).asSubclass(expectedType);
+            return clazz = Class.forName(name, true, classLoader).asSubclass(expectedType);
           } catch (ClassNotFoundException e1) {
             // ignore... assume first exception is best.
           }
         }
-    
-        throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Error loading class '" + cname + "'", e);
+
+        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, name +" Error loading class '" + cname + "'", e);
       }
-      
+
     } finally {
       if (clazz != null) {
         //cache the shortname vs FQN if it is loaded by the webapp classloader  and it is loaded
         // using a shortname
         if (clazz.getClassLoader() == SolrResourceLoader.class.getClassLoader() &&
-              !cname.equals(clazz.getName()) &&
-              (subpackages.length == 0 || subpackages == packages)) {
+            !cname.equals(clazz.getName()) &&
+            (subpackages.length == 0 || subpackages == packages)) {
           //store in the cache
           classNameCache.put(cname, clazz.getName());
         }
-        
+
         // print warning if class is deprecated
         if (clazz.isAnnotationPresent(Deprecated.class)) {
           log.warn("Solr loaded a deprecated plugin/analysis class [{}]. Please consult documentation how to replace it accordingly.",
@@ -582,9 +603,9 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
       }
     }
   }
-  
+
   static final String empty[] = new String[0];
-  
+
   @Override
   public <T> T newInstance(String name, Class<T> expectedType) {
     return newInstance(name, expectedType, empty);
@@ -593,33 +614,32 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
   private static final Class[] NO_CLASSES = new Class[0];
   private static final Object[] NO_OBJECTS = new Object[0];
 
-  public <T> T newInstance(String cname, Class<T> expectedType, String ... subpackages) {
+  public <T> T newInstance(String cname, Class<T> expectedType, String... subpackages) {
     return newInstance(cname, expectedType, subpackages, NO_CLASSES, NO_OBJECTS);
   }
 
-  public CoreAdminHandler newAdminHandlerInstance(final CoreContainer coreContainer, String cname, String ... subpackages) {
+  public CoreAdminHandler newAdminHandlerInstance(final CoreContainer coreContainer, String cname, String... subpackages) {
     Class<? extends CoreAdminHandler> clazz = findClass(cname, CoreAdminHandler.class, subpackages);
-    if( clazz == null ) {
-      throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
-          "Can not find class: "+cname + " in " + classLoader);
+    if (clazz == null) {
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+          "Can not find class: " + cname + " in " + classLoader);
     }
-    
+
     CoreAdminHandler obj = null;
     try {
       Constructor<? extends CoreAdminHandler> ctor = clazz.getConstructor(CoreContainer.class);
       obj = ctor.newInstance(coreContainer);
-    } 
-    catch (Exception e) {
-      throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
-          "Error instantiating class: '" + clazz.getName()+"'", e);
+    } catch (Exception e) {
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+          "Error instantiating class: '" + clazz.getName() + "'", e);
     }
 
     if (!live) {
       //TODO: Does SolrCoreAware make sense here since in a multi-core context
       // which core are we talking about ?
-      if( obj instanceof ResourceLoaderAware ) {
-        assertAwareCompatibility( ResourceLoaderAware.class, obj );
-        waitingForResources.add( (ResourceLoaderAware)obj );
+      if (obj instanceof ResourceLoaderAware) {
+        assertAwareCompatibility(ResourceLoaderAware.class, obj);
+        waitingForResources.add((ResourceLoaderAware) obj);
       }
     }
 
@@ -627,12 +647,11 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
   }
 
 
-
-  public <T> T newInstance(String cName, Class<T> expectedType, String [] subPackages, Class[] params, Object[] args){
+  public <T> T newInstance(String cName, Class<T> expectedType, String[] subPackages, Class[] params, Object[] args) {
     Class<? extends T> clazz = findClass(cName, expectedType, subPackages);
-    if( clazz == null ) {
-      throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
-          "Can not find class: "+cName + " in " + classLoader);
+    if (clazz == null) {
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+          "Can not find class: " + cName + " in " + classLoader);
     }
 
     T obj = null;
@@ -653,25 +672,25 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
       }
 
     } catch (Error err) {
-      log.error("Loading Class " + cName + " ("+clazz.getName() + ") triggered serious java error: "
-                + err.getClass().getName(), err);
+      log.error("Loading Class " + cName + " (" + clazz.getName() + ") triggered serious java error: "
+          + err.getClass().getName(), err);
       throw err;
 
     } catch (Exception e) {
-      throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
-          "Error instantiating class: '" + clazz.getName()+"'", e);
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+          "Error instantiating class: '" + clazz.getName() + "'", e);
     }
 
     if (!live) {
-      if( obj instanceof SolrCoreAware ) {
-        assertAwareCompatibility( SolrCoreAware.class, obj );
-        waitingForCore.add( (SolrCoreAware)obj );
+      if (obj instanceof SolrCoreAware) {
+        assertAwareCompatibility(SolrCoreAware.class, obj);
+        waitingForCore.add((SolrCoreAware) obj);
       }
-      if( obj instanceof ResourceLoaderAware ) {
-        assertAwareCompatibility( ResourceLoaderAware.class, obj );
-        waitingForResources.add( (ResourceLoaderAware)obj );
+      if (obj instanceof ResourceLoaderAware) {
+        assertAwareCompatibility(ResourceLoaderAware.class, obj);
+        waitingForResources.add((ResourceLoaderAware) obj);
       }
-      if (obj instanceof SolrInfoBean){
+      if (obj instanceof SolrInfoBean) {
         //TODO: Assert here?
         infoMBeans.add((SolrInfoBean) obj);
       }
@@ -680,12 +699,11 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
     return obj;
   }
 
-  
+
   /**
    * Tell all {@link SolrCoreAware} instances about the SolrCore
    */
-  public void inform(SolrCore core) 
-  {
+  public void inform(SolrCore core) {
     this.dataDir = core.getDataDir();
 
     // make a copy to avoid potential deadlock of a callback calling newInstance and trying to
@@ -698,22 +716,21 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
         waitingForCore.clear();
       }
 
-      for( SolrCoreAware aware : arr) {
-        aware.inform( core );
+      for (SolrCoreAware aware : arr) {
+        aware.inform(core);
       }
     }
 
     // this is the last method to be called in SolrCore before the latch is released.
     live = true;
   }
-  
+
   /**
    * Tell all {@link ResourceLoaderAware} instances about the loader
    */
-  public void inform( ResourceLoader loader ) throws IOException
-  {
+  public void inform(ResourceLoader loader) throws IOException {
 
-     // make a copy to avoid potential deadlock of a callback adding to the list
+    // make a copy to avoid potential deadlock of a callback adding to the list
     ResourceLoaderAware[] arr;
 
     while (waitingForResources.size() > 0) {
@@ -722,7 +739,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
         waitingForResources.clear();
       }
 
-      for( ResourceLoaderAware aware : arr) {
+      for (ResourceLoaderAware aware : arr) {
         aware.inform(loader);
       }
     }
@@ -730,6 +747,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
 
   /**
    * Register any {@link SolrInfoBean}s
+   *
    * @param infoRegistry The Info Registry
    */
   public void inform(Map<String, SolrInfoBean> infoRegistry) {
@@ -755,7 +773,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
       }
     }
   }
-  
+
   /**
    * Determines the solrhome from the environment.
    * Tries JNDI (java:comp/env/solr/home) then system property (solr.solr.home);
@@ -765,12 +783,13 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
   /**
    * Finds the solrhome based on looking up the value in one of three places:
    * <ol>
-   *  <li>JNDI: via java:comp/env/solr/home</li>
-   *  <li>The system property solr.solr.home</li>
-   *  <li>Look in the current working directory for a solr/ directory</li> 
+   * <li>JNDI: via java:comp/env/solr/home</li>
+   * <li>The system property solr.solr.home</li>
+   * <li>Look in the current working directory for a solr/ directory</li>
    * </ol>
-   *
+   * <p>
    * The return value is normalized.  Normalization essentially means it ends in a trailing slash.
+   *
    * @return A normalized solrhome
    * @see #normalizeDir(String)
    */
@@ -780,27 +799,27 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
     // Try JNDI
     try {
       Context c = new InitialContext();
-      home = (String)c.lookup("java:comp/env/"+project+"/home");
-      logOnceInfo("home_using_jndi", "Using JNDI solr.home: "+home );
+      home = (String) c.lookup("java:comp/env/" + project + "/home");
+      logOnceInfo("home_using_jndi", "Using JNDI solr.home: " + home);
     } catch (NoInitialContextException e) {
-      log.debug("JNDI not configured for "+project+" (NoInitialContextEx)");
+      log.debug("JNDI not configured for " + project + " (NoInitialContextEx)");
     } catch (NamingException e) {
-      log.debug("No /"+project+"/home in JNDI");
-    } catch( RuntimeException ex ) {
+      log.debug("No /" + project + "/home in JNDI");
+    } catch (RuntimeException ex) {
       log.warn("Odd RuntimeException while testing for JNDI: " + ex.getMessage());
-    } 
-    
+    }
+
     // Now try system property
-    if( home == null ) {
+    if (home == null) {
       String prop = project + ".solr.home";
       home = System.getProperty(prop);
-      if( home != null ) {
-        logOnceInfo("home_using_sysprop", "Using system property "+prop+": " + home );
+      if (home != null) {
+        logOnceInfo("home_using_sysprop", "Using system property " + prop + ": " + home);
       }
     }
-    
+
     // if all else fails, try 
-    if( home == null ) {
+    if (home == null) {
       home = project + '/';
       logOnceInfo("home_default", project + " home defaulted to '" + home + "' (could not find system property or JNDI)");
     }
@@ -809,22 +828,23 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
 
   /**
    * Solr allows users to store arbitrary files in a special directory located directly under SOLR_HOME.
-   *
+   * <p>
    * This directory is generally created by each node on startup.  Files located in this directory can then be
    * manipulated using select Solr features (e.g. streaming expressions).
    */
   public static final String USER_FILES_DIRECTORY = "userfiles";
+
   public static void ensureUserFilesDataDir(Path solrHome) {
     final Path userFilesPath = getUserFilesPath(solrHome);
     final File userFilesDirectory = new File(userFilesPath.toString());
-    if (! userFilesDirectory.exists()) {
+    if (!userFilesDirectory.exists()) {
       try {
         final boolean created = userFilesDirectory.mkdir();
-        if (! created) {
+        if (!created) {
           log.warn("Unable to create [{}] directory in SOLR_HOME [{}].  Features requiring this directory may fail.", USER_FILES_DIRECTORY, solrHome);
         }
       } catch (Exception e) {
-          log.warn("Unable to create [" + USER_FILES_DIRECTORY + "] directory in SOLR_HOME [" + solrHome + "].  Features requiring this directory may fail.", e);
+        log.warn("Unable to create [" + USER_FILES_DIRECTORY + "] directory in SOLR_HOME [" + solrHome + "].  Features requiring this directory may fail.", e);
       }
     }
   }
@@ -847,72 +867,73 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
   public Path getInstancePath() {
     return instanceDir;
   }
-  
+
   /**
    * Keep a list of classes that are allowed to implement each 'Aware' interface
    */
   private static final Map<Class, Class[]> awareCompatibility;
+
   static {
     awareCompatibility = new HashMap<>();
-    awareCompatibility.put( 
-      SolrCoreAware.class, new Class[] {
-        // DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY
-        // VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
-        CodecFactory.class,
-        DirectoryFactory.class,
-        ManagedIndexSchemaFactory.class,
-        QueryResponseWriter.class,
-        SearchComponent.class,
-        ShardHandlerFactory.class,
-        SimilarityFactory.class,
-        SolrRequestHandler.class,
-        UpdateRequestProcessorFactory.class
-      }
+    awareCompatibility.put(
+        SolrCoreAware.class, new Class[]{
+            // DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY
+            // VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
+            CodecFactory.class,
+            DirectoryFactory.class,
+            ManagedIndexSchemaFactory.class,
+            QueryResponseWriter.class,
+            SearchComponent.class,
+            ShardHandlerFactory.class,
+            SimilarityFactory.class,
+            SolrRequestHandler.class,
+            UpdateRequestProcessorFactory.class
+        }
     );
 
     awareCompatibility.put(
-      ResourceLoaderAware.class, new Class[] {
-        // DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY
-        // VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
-        CharFilterFactory.class,
-        TokenFilterFactory.class,
-        TokenizerFactory.class,
-        QParserPlugin.class,
-        FieldType.class
-      }
+        ResourceLoaderAware.class, new Class[]{
+            // DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY
+            // VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
+            CharFilterFactory.class,
+            TokenFilterFactory.class,
+            TokenizerFactory.class,
+            QParserPlugin.class,
+            FieldType.class
+        }
     );
   }
 
   /**
    * Utility function to throw an exception if the class is invalid
    */
-  static void assertAwareCompatibility( Class aware, Object obj )
-  {
-    Class[] valid = awareCompatibility.get( aware );
-    if( valid == null ) {
-      throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
-          "Unknown Aware interface: "+aware );
-    }
-    for( Class v : valid ) {
-      if( v.isInstance( obj ) ) {
+  static void assertAwareCompatibility(Class aware, Object obj) {
+    Class[] valid = awareCompatibility.get(aware);
+    if (valid == null) {
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+          "Unknown Aware interface: " + aware);
+    }
+    for (Class v : valid) {
+      if (v.isInstance(obj)) {
         return;
       }
     }
     StringBuilder builder = new StringBuilder();
-    builder.append( "Invalid 'Aware' object: " ).append( obj );
-    builder.append( " -- ").append( aware.getName() );
-    builder.append(  " must be an instance of: " );
-    for( Class v : valid ) {
-      builder.append( "[" ).append( v.getName() ).append( "] ") ;
+    builder.append("Invalid 'Aware' object: ").append(obj);
+    builder.append(" -- ").append(aware.getName());
+    builder.append(" must be an instance of: ");
+    for (Class v : valid) {
+      builder.append("[").append(v.getName()).append("] ");
     }
-    throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, builder.toString() );
+    throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, builder.toString());
   }
 
   @Override
   public void close() throws IOException {
     IOUtils.close(classLoader);
   }
-  public List<SolrInfoBean> getInfoMBeans(){
+
+  public List<SolrInfoBean> getInfoMBeans() {
     return Collections.unmodifiableList(infoMBeans);
   }
 
@@ -922,8 +943,8 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
     File confFile = new File(loader.getConfigDir(), resourceName);
     try {
       File parentDir = confFile.getParentFile();
-      if ( ! parentDir.isDirectory()) {
-        if ( ! parentDir.mkdirs()) {
+      if (!parentDir.isDirectory()) {
+        if (!parentDir.mkdirs()) {
           final String msg = "Can't create managed schema directory " + parentDir.getAbsolutePath();
           log.error(msg);
           throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg);
diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
index 11c6404..2085221 100644
--- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
@@ -68,6 +68,7 @@ import org.apache.solr.core.RequestParams;
 import org.apache.solr.core.SolrConfig;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.pkg.PackageListeners;
 import org.apache.solr.request.LocalSolrQueryRequest;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.request.SolrRequestHandler;
@@ -245,8 +246,22 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
             if (componentName != null) {
               Map map = (Map) val.get(parts.get(1));
               if (map != null) {
-                val.put(parts.get(1), makeMap(componentName, map.get(componentName)));
+                Object o = map.get(componentName);
+                val.put(parts.get(1), makeMap(componentName, o));
+                if(req.getParams().getBool("meta", false)){
+                  for (PackageListeners.Listener listener :
+                      req.getCore().getPackageListeners().getListeners()) {
+                    PluginInfo info = listener.pluginInfo();
+                    if(info.type.equals(parts.get(1)) && info.name.equals(componentName)){
+                      if (o instanceof Map) {
+                        Map m1 = (Map) o;
+                        m1.put("_packageinfo_", listener.getPackageVersion());
+                      }
+                    }
+                  }
+                }
               }
+
             }
 
             resp.add("config", val);
@@ -488,6 +503,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
       }
       List errs = CommandOperation.captureErrors(ops);
       if (!errs.isEmpty()) {
+        log.error("ERRROR:" +Utils.toJSONString(errs));
         throw new ApiBag.ExceptionWithErrObject(SolrException.ErrorCode.BAD_REQUEST,"error processing commands", errs);
       }
 
@@ -495,7 +511,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
       if (loader instanceof ZkSolrResourceLoader) {
         int latestVersion = ZkController.persistConfigResourceToZooKeeper((ZkSolrResourceLoader) loader, overlay.getZnodeVersion(),
             ConfigOverlay.RESOURCE_NAME, overlay.toByteArray(), true);
-        log.info("Executed config commands successfully and persisted to ZK {}", ops);
+        log.debug("Executed config commands successfully and persisted to ZK {}", ops);
         waitForAllReplicasState(req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName(),
             req.getCore().getCoreContainer().getZkController(),
             ConfigOverlay.NAME,
@@ -503,7 +519,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
       } else {
         SolrResourceLoader.persistConfLocally(loader, ConfigOverlay.RESOURCE_NAME, overlay.toByteArray());
         req.getCore().getCoreContainer().reload(req.getCore().getName());
-        log.info("Executed config commands successfully and persited to File System {}", ops);
+        log.debug("Executed config commands successfully and persited to File System {}", ops);
       }
 
     }
@@ -570,6 +586,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
         try {
           req.getCore().createInitInstance(new PluginInfo(SolrRequestHandler.TYPE, op.getDataMap()), expected, clz, "");
         } catch (Exception e) {
+          log.error("Error checking plugin : ",e);
           op.addError(e.getMessage());
           return false;
         }
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
new file mode 100644
index 0000000..0267f37
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java
@@ -0,0 +1,367 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.pkg;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.solr.api.Command;
+import org.apache.solr.api.EndPoint;
+import org.apache.solr.api.PayloadObj;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.cloud.ZooKeeperException;
+import org.apache.solr.common.util.CommandOperation;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.filestore.PackageStoreAPI;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.util.ReflectMapWriter;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH;
+import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_EDIT_PERM;
+import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_PERM;
+
+public class PackageAPI {
+  public static final String PACKAGES = "packages";
+  public final boolean enablePackages = Boolean.parseBoolean(System.getProperty("enable.packages", "false"));
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  final CoreContainer coreContainer;
+  private ObjectMapper mapper = new ObjectMapper();
+  private final PackageLoader packageLoader;
+  Packages pkgs;
+
+  public final Edit editAPI = new Edit();
+  public final Read readAPI = new Read();
+
+  public PackageAPI(CoreContainer coreContainer, PackageLoader loader) {
+    this.coreContainer = coreContainer;
+    this.packageLoader = loader;
+    pkgs = new Packages();
+    SolrZkClient zkClient = coreContainer.getZkController().getZkClient();
+    try {
+      registerListener(zkClient);
+    } catch (KeeperException | InterruptedException e) {
+      e.printStackTrace();
+    }
+  }
+
+  private void registerListener(SolrZkClient zkClient)
+      throws KeeperException, InterruptedException {
+    String path = SOLR_PKGS_PATH;
+    zkClient.exists(path,
+        new Watcher() {
+
+          @Override
+          public void process(WatchedEvent event) {
+            // session events are not change events, and do not remove the watcher
+            if (Event.EventType.None.equals(event.getType())) {
+              return;
+            }
+            try {
+              synchronized (this) {
+                log.debug("Updating [{}] ... ", path);
+
+                // remake watch
+                final Watcher thisWatch = this;
+                final Stat stat = new Stat();
+                final byte[] data = zkClient.getData(path, thisWatch, stat, true);
+                pkgs = readPkgsFromZk(data, stat);
+                packageLoader.refreshPackageConf();
+              }
+            } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) {
+              log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: [{}]", e.getMessage());
+            } catch (KeeperException e) {
+              log.error("A ZK error has occurred", e);
+              throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
+            } catch (InterruptedException e) {
+              // Restore the interrupted status
+              Thread.currentThread().interrupt();
+              log.warn("Interrupted", e);
+            }
+          }
+
+        }, true);
+  }
+
+
+  private Packages readPkgsFromZk(byte[] data, Stat stat) throws KeeperException, InterruptedException {
+
+    if (data == null || stat == null) {
+      stat = new Stat();
+      data = coreContainer.getZkController().getZkClient()
+          .getData(ZkStateReader.CLUSTER_PROPS, null, stat, true);
+
+    }
+    Packages packages = null;
+    if (data == null || data.length == 0) {
+      packages = new Packages();
+    } else {
+      try {
+        packages = mapper.readValue(data, Packages.class);
+        packages.znodeVersion = stat.getVersion();
+      } catch (IOException e) {
+        //invalid data in packages
+        //TODO handle properly;
+        return new Packages();
+      }
+    }
+    return packages;
+  }
+
+
+  public static class Packages implements ReflectMapWriter {
+    @JsonProperty
+    public int znodeVersion = -1;
+
+    @JsonProperty
+    public Map<String, List<PkgVersion>> packages = new LinkedHashMap<>();
+
+
+    public Packages copy() {
+      Packages p = new Packages();
+      p.znodeVersion = this.znodeVersion;
+      p.packages = new LinkedHashMap<>();
+      packages.forEach((s, versions) ->
+          p.packages.put(s, new ArrayList<>(versions)));
+      return p;
+    }
+  }
+
+  public static class PkgVersion implements ReflectMapWriter {
+
+    @JsonProperty
+    public String version;
+
+    @JsonProperty
+    public List<String> files;
+
+    public PkgVersion() {
+    }
+
+    public PkgVersion(AddVersion addVersion) {
+      this.version = addVersion.version;
+      this.files = addVersion.files;
+    }
+
+
+    @Override
+    public boolean equals(Object obj) {
+      if (obj instanceof PkgVersion) {
+        PkgVersion that = (PkgVersion) obj;
+        return Objects.equals(this.version, that.version)
+            && Objects.equals(this.files, that.files);
+
+      }
+      return false;
+    }
+  }
+
+
+  @EndPoint(method = SolrRequest.METHOD.POST,
+      path = "/cluster/package",
+      permission = PACKAGE_EDIT_PERM)
+  public class Edit {
+
+    @Command(name = "add")
+    public void add(SolrQueryRequest req, SolrQueryResponse rsp, PayloadObj<AddVersion> payload) {
+      if (!checkEnabled(payload)) return;
+      AddVersion add = payload.get();
+      if (add.files.isEmpty()) {
+        payload.addError("No files specified");
+        return;
+      }
+      PackageStoreAPI packageStoreAPI = coreContainer.getPackageStoreAPI();
+      packageStoreAPI.validateFiles(add.files, true, s -> payload.addError(s));
+      if (payload.hasError()) return;
+      Packages[] finalState = new Packages[1];
+      try {
+        coreContainer.getZkController().getZkClient().atomicUpdate(SOLR_PKGS_PATH, (stat, bytes) -> {
+          Packages packages = null;
+          try {
+            packages = bytes == null ? new Packages() : mapper.readValue(bytes, Packages.class);
+            packages = packages.copy();
+          } catch (IOException e) {
+            log.error("Error deserializing packages.json", e);
+            packages = new Packages();
+          }
+          packages.packages.computeIfAbsent(add.pkg, Utils.NEW_ARRAYLIST_FUN).add(new PkgVersion(add));
+          packages.znodeVersion = stat.getVersion() + 1;
+          finalState[0] = packages;
+          return Utils.toJSON(packages);
+        });
+      } catch (KeeperException | InterruptedException e) {
+        finalState[0] = null;
+        handleZkErr(e);
+      }
+      if (finalState[0] != null) {
+//        succeeded in updating
+        pkgs = finalState[0];
+        notifyAllNodesToSync(pkgs.znodeVersion);
+        packageLoader.refreshPackageConf();
+      }
+
+    }
+
+    @Command(name = "delete")
+    public void del(SolrQueryRequest req, SolrQueryResponse rsp, PayloadObj<DelVersion> payload) {
+      if (!checkEnabled(payload)) return;
+      DelVersion delVersion = payload.get();
+      try {
+        coreContainer.getZkController().getZkClient().atomicUpdate(SOLR_PKGS_PATH, (stat, bytes) -> {
+          Packages packages = null;
+          try {
+            packages = mapper.readValue(bytes, Packages.class);
+            packages = packages.copy();
+          } catch (IOException e) {
+            packages = new Packages();
+          }
+
+          List<PkgVersion> versions = packages.packages.get(delVersion.pkg);
+          if (versions == null || versions.isEmpty()) {
+            payload.addError("No such package: " + delVersion.pkg);
+            return null;// no change
+          }
+          int idxToremove = -1;
+          for (int i = 0; i < versions.size(); i++) {
+            if (Objects.equals(versions.get(i).version, delVersion.version)) {
+              idxToremove = i;
+              break;
+            }
+          }
+          if (idxToremove == -1) {
+            payload.addError("No such version: " + delVersion.version);
+            return null;
+          }
+          versions.remove(idxToremove);
+          packages.znodeVersion = stat.getVersion() + 1;
+          return Utils.toJSON(packages);
+        });
+      } catch (KeeperException | InterruptedException e) {
+        handleZkErr(e);
+
+      }
+
+
+    }
+
+  }
+
+  private boolean checkEnabled(CommandOperation payload) {
+    if (!enablePackages) {
+      payload.addError("Package loading is not enabled , Start your nodes with -Denable.packages=true");
+      return false;
+    }
+    return true;
+  }
+
+  @EndPoint(
+      method = SolrRequest.METHOD.GET,
+      path = {"/cluster/package/",
+          "/cluster/package/{name}"},
+      permission = PACKAGE_READ_PERM
+  )
+  public class Read {
+    @Command()
+    public void get(SolrQueryRequest req, SolrQueryResponse rsp) {
+      int expectedVersion = req.getParams().getInt("expectedVersion", -1);
+      if (expectedVersion != -1) {
+        syncToVersion(expectedVersion);
+      }
+      String name = req.getPathTemplateValues().get("name");
+      if (name == null) {
+        rsp.add("result", pkgs);
+      } else {
+        rsp.add("result", Collections.singletonMap(name, pkgs.packages.get(name)));
+      }
+    }
+
+    private void syncToVersion(int expectedVersion) {
+      for (int i = 0; i < 10; i++) {
+        log.debug("my version is {} , and expected version {}", pkgs.znodeVersion, expectedVersion);
+        if (pkgs.znodeVersion >= expectedVersion) {
+          return;
+        }
+        try {
+          Thread.sleep(10);
+        } catch (InterruptedException e) {
+        }
+        try {
+          pkgs = readPkgsFromZk(null, null);
+        } catch (KeeperException | InterruptedException e) {
+          handleZkErr(e);
+
+        }
+
+      }
+
+    }
+
+
+  }
+
+  public static class AddVersion implements ReflectMapWriter {
+    @JsonProperty(value = "package", required = true)
+    public String pkg;
+    @JsonProperty(required = true)
+    public String version;
+    @JsonProperty(required = true)
+    public List<String> files;
+
+  }
+
+  public static class DelVersion implements ReflectMapWriter {
+    @JsonProperty(value = "package", required = true)
+    public String pkg;
+    @JsonProperty(required = true)
+    public String version;
+
+  }
+
+  void notifyAllNodesToSync(int expected) {
+    for (String s : coreContainer.getPackageStoreAPI().shuffledNodes()) {
+      Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(),
+          coreContainer.getZkController().zkStateReader.getBaseUrlForNodeName(s).replace("/solr", "/api") + "/cluster/package?wt=javabin&omitHeader=true&expectedVersion" + expected,
+          Utils.JAVABINCONSUMER);
+    }
+  }
+
+  public void handleZkErr(Exception e) {
+    log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e));
+  }
+
+
+}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java
new file mode 100644
index 0000000..c6ebae5
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.pkg;
+
+import java.lang.ref.WeakReference;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.solr.core.PluginInfo;
+
+public class PackageListeners {
+  // this registry only keeps a weak reference because it does not want to
+  // cause a memory leak if the listener forgets to unregister itself
+  private List<WeakReference<Listener>> listeners = new ArrayList<>();
+
+  public synchronized void addListener(Listener listener) {
+    listeners.add(new WeakReference<>(listener));
+
+  }
+
+  public synchronized void removeListener(Listener listener) {
+    Iterator<WeakReference<Listener>> it = listeners.iterator();
+    while (it.hasNext()) {
+      WeakReference<Listener> ref = it.next();
+      Listener pkgListener = ref.get();
+      if(pkgListener == null || pkgListener == listener){
+        it.remove();
+      }
+
+    }
+
+  }
+
+  synchronized void packagesUpdated(List<PackageLoader.Package> pkgs){
+    for (PackageLoader.Package pkgInfo : pkgs) {
+      invokeListeners(pkgInfo);
+    }
+  }
+
+  private synchronized void invokeListeners(PackageLoader.Package pkg) {
+    for (WeakReference<Listener> ref : listeners) {
+      Listener listener = ref.get();
+      if (listener != null && listener.packageName().equals(pkg.name())) {
+        listener.changed(pkg);
+      }
+    }
+  }
+
+  public List<Listener> getListeners(){
+    List<Listener> result = new ArrayList<>();
+    for (WeakReference<Listener> ref : listeners) {
+      Listener l = ref.get();
+      if(l != null){
+        result.add(l);
+      }
+
+    }
+    return result;
+  }
+
+
+
+  public interface Listener {
+    String packageName();
+
+    PluginInfo pluginInfo();
+
+    void changed(PackageLoader.Package pkg);
+
+    PackageLoader.Package.Version getPackageVersion();
+
+  }
+}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java
new file mode 100644
index 0000000..7efcc88
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.pkg;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.MalformedURLException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrResourceLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PackageLoader {
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  private final CoreContainer coreContainer;
+  private final Map<String, Package> packageClassLoaders = new ConcurrentHashMap<>();
+
+  private PackageAPI.Packages myCopy;
+
+  private PackageAPI packageAPI;
+
+
+  public PackageLoader(CoreContainer coreContainer) {
+    this.coreContainer = coreContainer;
+    packageAPI = new PackageAPI(coreContainer, this);
+    myCopy = packageAPI.pkgs;
+
+  }
+
+  public PackageAPI getPackageAPI() {
+    return packageAPI;
+  }
+
+  public Package getPackage(String key) {
+    return packageClassLoaders.get(key);
+  }
+
+  public Map<String, Package> getPackages() {
+    return Collections.EMPTY_MAP;
+  }
+
+  public void refreshPackageConf() {
+    log.info("{} updated to version {}", ZkStateReader.SOLR_PKGS_PATH, packageAPI.pkgs.znodeVersion);
+
+    List<Package> updated = new ArrayList<>();
+    Map<String, List<PackageAPI.PkgVersion>> modified = getModified(myCopy, packageAPI.pkgs);
+    for (Map.Entry<String, List<PackageAPI.PkgVersion>> e : modified.entrySet()) {
+      if (e.getValue() != null) {
+        Package p = packageClassLoaders.get(e.getKey());
+        if (e.getValue() != null && p == null) {
+          packageClassLoaders.put(e.getKey(), p = new Package(e.getKey()));
+        }
+        p.updateVersions(e.getValue());
+        updated.add(p);
+      } else {
+        Package p = packageClassLoaders.remove(e.getKey());
+        if (p != null) {
+          //other classes are holding to a reference to this objecec
+          // they should know that this is removed
+          p.markDeleted();
+        }
+      }
+    }
+    for (SolrCore core : coreContainer.getCores()) {
+      core.getPackageListeners().packagesUpdated(updated);
+
+    }
+  }
+
+  public Map<String, List<PackageAPI.PkgVersion>> getModified(PackageAPI.Packages old, PackageAPI.Packages newPkgs) {
+    Map<String, List<PackageAPI.PkgVersion>> changed = new HashMap<>();
+    for (Map.Entry<String, List<PackageAPI.PkgVersion>> e : newPkgs.packages.entrySet()) {
+      List<PackageAPI.PkgVersion> versions = old.packages.get(e.getKey());
+      if (versions != null) {
+        if (!Objects.equals(e.getValue(), versions)) {
+          changed.put(e.getKey(), e.getValue());
+        }
+      } else {
+        changed.put(e.getKey(), e.getValue());
+      }
+    }
+    //some packages are deleted altogether
+    for (String s : old.packages.keySet()) {
+      if (!newPkgs.packages.keySet().contains(s)) {
+        changed.put(s, null);
+      }
+    }
+
+    return changed;
+
+  }
+
+  public SolrResourceLoader getResourceLoader(String pkg, String version) {
+    return null;
+  }
+
+
+  public class Package {
+    final String name;
+    final Map<String, Version> myVersions = new ConcurrentHashMap<>();
+    private List<String> sortedVersions = new CopyOnWriteArrayList<>();
+    String latest;
+    private boolean deleted;
+
+
+    public Package(String name) {
+      this.name = name;
+    }
+
+    public boolean isDeleted() {
+      return deleted;
+    }
+
+
+    private synchronized void updateVersions(List<PackageAPI.PkgVersion> modified) {
+      for (PackageAPI.PkgVersion v : modified) {
+        Version version = myVersions.get(v.version);
+        if (version == null) {
+          myVersions.put(v.version, new Version(this, v));
+          sortedVersions.add(v.version);
+        }
+      }
+
+      Set<String> newVersions = new HashSet<>();
+      for (PackageAPI.PkgVersion v : modified) {
+        newVersions.add(v.version);
+      }
+      for (String s : new HashSet<>(myVersions.keySet())) {
+        if (!newVersions.contains(s)) {
+          sortedVersions.remove(s);
+          myVersions.remove(s);
+        }
+      }
+
+      sortedVersions.sort(String::compareTo);
+      if (sortedVersions.size() > 0) {
+        latest = sortedVersions.get(sortedVersions.size() - 1);
+      } else {
+        latest = null;
+      }
+
+    }
+
+
+    public Version getLatest() {
+      return latest == null ? null : myVersions.get(latest);
+    }
+
+    public Version getLatest(String lessThan) {
+      String latest = null;
+      for (String v : (ArrayList<String>) new ArrayList(sortedVersions)) {
+        if (v.compareTo(lessThan) < 1) {
+          latest = v;
+        } else break;
+
+      }
+
+
+      return latest == null ? null : myVersions.get(latest);
+    }
+
+    public String name() {
+      return name;
+    }
+
+    private void markDeleted() {
+      deleted = true;
+      myVersions.clear();
+      sortedVersions.clear();
+      latest = null;
+
+    }
+
+    public class Version implements MapWriter {
+      private final Package parent;
+      private SolrResourceLoader loader;
+
+      private final PackageAPI.PkgVersion version;
+
+      @Override
+      public void writeMap(EntryWriter ew) throws IOException {
+        ew.put("package", parent.name());
+        version.writeMap(ew);
+      }
+
+      public Version(Package parent, PackageAPI.PkgVersion v) {
+        this.parent = parent;
+        this.version = v;
+        List<Path> paths = new ArrayList<>();
+        for (String file : version.files) {
+          paths.add(coreContainer.getPackageStoreAPI().getPackageStore().getRealpath(file));
+        }
+
+        try {
+          loader = new SolrResourceLoader(
+              "PACKAGE_LOADER:"+ parent.name()+ ":"+ version,
+              paths,
+              coreContainer.getResourceLoader().getInstancePath(),
+              coreContainer.getResourceLoader().getClassLoader());
+        } catch (MalformedURLException e) {
+          log.error("Could not load classloader ", e);
+        }
+      }
+
+      public String getVersion() {
+        return version.version;
+      }
+
+      public Collection getFiles() {
+        return Collections.unmodifiableList(version.files);
+      }
+
+      public SolrResourceLoader getLoader() {
+        return loader;
+
+      }
+
+    }
+  }
+
+
+}
diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
new file mode 100644
index 0000000..f0364c4
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.pkg;
+
+import java.lang.invoke.MethodHandles;
+
+import org.apache.solr.core.PluginBag;
+import org.apache.solr.core.PluginInfo;
+import org.apache.solr.core.SolrConfig;
+import org.apache.solr.core.SolrCore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PackagePluginHolder<T> extends PluginBag.PluginHolder<T> {
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  private final SolrCore core;
+  private final SolrConfig.SolrPluginInfo pluginMeta;
+  private PackageLoader.Package aPackage;
+  private PackageLoader.Package.Version pkgVersion;
+
+
+  public PackagePluginHolder(PluginInfo info, SolrCore core, SolrConfig.SolrPluginInfo pluginMeta) {
+    super(info);
+    this.core = core;
+    this.pluginMeta = pluginMeta;
+
+    reload(aPackage = core.getCoreContainer().getPackageLoader().getPackage(info.pkgName));
+    core.getPackageListeners().addListener(new PackageListeners.Listener() {
+      @Override
+      public String packageName() {
+        return info.pkgName;
+      }
+
+      @Override
+      public PluginInfo pluginInfo() {
+        return info;
+      }
+
+      @Override
+      public void changed(PackageLoader.Package pkg) {
+        reload(pkg);
+
+      }
+
+      @Override
+      public PackageLoader.Package.Version getPackageVersion() {
+        return pkgVersion;
+      }
+
+    });
+  }
+
+
+  private synchronized void reload(PackageLoader.Package pkg) {
+    if(pkgVersion != null && aPackage.getLatest() == pkgVersion ) return;
+
+    if (inst != null) log.info("reloading plugin {} ", pluginInfo.name);
+    PackageLoader.Package.Version newest = pkg.getLatest();
+    if(newest == null) return;
+    Object instance = SolrCore.createInstance(pluginInfo.className,
+        pluginMeta.clazz, pluginMeta.getCleanTag(), core, newest.getLoader());
+    PluginBag.initInstance(instance, pluginInfo);
+    T old = inst;
+    inst = (T) instance;
+    pkgVersion = newest;
+    if (old instanceof AutoCloseable) {
+      AutoCloseable closeable = (AutoCloseable) old;
+      try {
+        closeable.close();
+      } catch (Exception e) {
+        log.error("error closing plugin", e);
+      }
+    }
+  }
+
+}
\ No newline at end of file
diff --git a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
index a4c7c0d..b5d4099 100644
--- a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
+++ b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java
@@ -53,6 +53,9 @@ public interface PermissionNameProvider {
     METRICS_HISTORY_READ_PERM("metrics-history-read", null),
     FILESTORE_READ_PERM("filestore-read", null),
     FILESTORE_WRITE_PERM("filestore-write", null),
+    PACKAGE_EDIT_PERM("package-edit", null),
+    PACKAGE_READ_PERM("package-read", null),
+
     ALL("all", unmodifiableSet(new HashSet<>(asList("*", null))))
     ;
     final String name;
diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
new file mode 100644
index 0000000..5de69cf
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java
@@ -0,0 +1,391 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.pkg;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.request.V2Request;
+import org.apache.solr.cloud.ConfigRequest;
+import org.apache.solr.cloud.MiniSolrCloudCluster;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.MapWriterMap;
+import org.apache.solr.common.NavigableObject;
+import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.filestore.TestDistribPackageStore;
+import org.apache.solr.util.LogLevel;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.data.Stat;
+import org.junit.Test;
+
+import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH;
+import static org.apache.solr.common.params.CommonParams.JAVABIN;
+import static org.apache.solr.common.params.CommonParams.WT;
+import static org.apache.solr.core.TestDynamicLoading.getFileContent;
+import static org.apache.solr.filestore.TestDistribPackageStore.readFile;
+
+@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG")
+public class TestPackages extends SolrCloudTestCase {
+
+  @Test
+  public void testPluginLoading() throws Exception {
+    System.setProperty("enable.packages", "true");
+    MiniSolrCloudCluster cluster =
+        configureCluster(4)
+            .withJettyConfig(jetty -> jetty.enableV2(true))
+            .addConfig("conf", configset("cloud-minimal"))
+            .configure();
+    try {
+      String FILE1 = "/mypkg/runtimelibs.jar";
+      String FILE2 = "/mypkg/runtimelibs_v2.jar";
+      String COLLECTION_NAME = "testPluginLoadingColl";
+      byte[] derFile = readFile("cryptokeys/pub_key512.der");
+      cluster.getZkClient().makePath("/keys/exe", true);
+      cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
+      postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1,
+          "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ==");
+
+      PackageAPI.AddVersion add = new PackageAPI.AddVersion();
+      add.version = "1.0";
+      add.pkg = "mypkg";
+      add.files = Arrays.asList(new String[]{FILE1});
+      V2Request req = new V2Request.Builder("/cluster/package")
+          .forceV2(true)
+          .withMethod(SolrRequest.METHOD.POST)
+          .withPayload(Collections.singletonMap("add", add))
+          .build();
+
+      req.process(cluster.getSolrClient());
+
+
+      CollectionAdminRequest
+          .createCollection(COLLECTION_NAME, "conf", 2, 2)
+          .setMaxShardsPerNode(100)
+          .process(cluster.getSolrClient());
+      cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4);
+
+      TestDistribPackageStore.assertResponseValues(10,
+          () -> new V2Request.Builder("/cluster/package").
+              withMethod(SolrRequest.METHOD.GET)
+              .build().process(cluster.getSolrClient()),
+          Utils.makeMap(
+              ":result:packages:mypkg[0]:version", "1.0",
+              ":result:packages:mypkg[0]:files[0]", FILE1
+          ));
+
+      String payload = "{\n" +
+          "'create-requesthandler' : { 'name' : '/runtime', 'class': 'mypkg:org.apache.solr.core.RuntimeLibReqHandler' }," +
+          "'create-searchcomponent' : { 'name' : 'get', 'class': 'mypkg:org.apache.solr.core.RuntimeLibSearchComponent'  }," +
+          "'create-queryResponseWriter' : { 'name' : 'json1', 'class': 'mypkg:org.apache.solr.core.RuntimeLibResponseWriter' }" +
+          "}";
+      cluster.getSolrClient().request(new ConfigRequest(payload) {
+        @Override
+        public String getCollection() {
+          return COLLECTION_NAME;
+        }
+      });
+
+      verifyCmponent(cluster.getSolrClient(),
+          COLLECTION_NAME, "queryResponseWriter", "json1",
+          "mypkg", "1.0" );
+
+      verifyCmponent(cluster.getSolrClient(),
+          COLLECTION_NAME, "searchComponent", "get",
+          "mypkg", "1.0" );
+
+      verifyCmponent(cluster.getSolrClient(),
+          COLLECTION_NAME, "requestHandler", "/runtime",
+          "mypkg", "1.0" );
+
+
+
+      executeReq( "/" + COLLECTION_NAME + "/runtime?wt=javabin", cluster.getRandomJetty(random()),
+          Utils.JAVABINCONSUMER,
+          Utils.makeMap("class", "org.apache.solr.core.RuntimeLibReqHandler"));
+
+      executeReq( "/" + COLLECTION_NAME + "/get?wt=json", cluster.getRandomJetty(random()),
+          Utils.JSONCONSUMER,
+          Utils.makeMap("class", "org.apache.solr.core.RuntimeLibSearchComponent",
+              "Version","1"));
+
+
+      executeReq( "/" + COLLECTION_NAME + "/runtime?wt=json1", cluster.getRandomJetty(random()),
+          Utils.JSONCONSUMER,
+          Utils.makeMap("wt", "org.apache.solr.core.RuntimeLibResponseWriter"));
+
+      //now upload the second jar
+      postFileAndWait(cluster, "runtimecode/runtimelibs_v2.jar.bin", FILE2,
+          "j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA==");
+
+      //add the version using package API
+      add.version = "1.1";
+      add.pkg = "mypkg";
+      add.files = Arrays.asList(new String[]{FILE2});
+      req.process(cluster.getSolrClient());
+
+      verifyCmponent(cluster.getSolrClient(),
+          COLLECTION_NAME, "queryResponseWriter", "json1",
+          "mypkg", "1.1" );
+
+      verifyCmponent(cluster.getSolrClient(),
+          COLLECTION_NAME, "searchComponent", "get",
+          "mypkg", "1.1" );
+
+      verifyCmponent(cluster.getSolrClient(),
+          COLLECTION_NAME, "requestHandler", "/runtime",
+          "mypkg", "1.1" );
+
+      /*executeReq( "/" + COLLECTION_NAME + "/get?wt=json", cluster.getRandomJetty(random()),
+          Utils.JSONCONSUMER,
+          Utils.makeMap("class", "org.apache.solr.core.RuntimeLibSearchComponent",
+              "Version","2"));
+
+      PackageAPI.DelVersion delVersion = new PackageAPI.DelVersion();
+      delVersion.pkg = "mypkg";
+      delVersion.version = "1.1";
+      new V2Request.Builder("/cluster/package")
+          .withMethod(SolrRequest.METHOD.POST)
+          .forceV2(true)
+          .withPayload(delVersion)
+          .build()
+          .process(cluster.getSolrClient());
+
+      verifyCmponent(cluster.getSolrClient(),
+          COLLECTION_NAME, "queryResponseWriter", "json1",
+          "mypkg", "1.0" );
+
+      verifyCmponent(cluster.getSolrClient(),
+          COLLECTION_NAME, "searchComponent", "get",
+          "mypkg", "1.0" );
+
+      verifyCmponent(cluster.getSolrClient(),
+          COLLECTION_NAME, "requestHandler", "/runtime",
+          "mypkg", "1.0" );
+
+*/
+    } finally {
+      cluster.shutdown();
+    }
+
+  }
+
+  private void executeReq(String uri, JettySolrRunner jetty, Utils.InputStreamConsumer parser, Map expected) throws Exception {
+    try(HttpSolrClient client = (HttpSolrClient) jetty.newClient()){
+      TestDistribPackageStore.assertResponseValues(10,
+          () -> {
+            Object o = Utils.executeGET(client.getHttpClient(),
+                jetty.getBaseUrl() + uri, parser);
+            if(o instanceof NavigableObject) return (NavigableObject) o;
+            if(o instanceof Map) return new MapWriterMap((Map) o);
+            throw new RuntimeException("Unknown response");
+          }, expected);
+
+    }
+  }
+
+  private void verifyCmponent(SolrClient client, String COLLECTION_NAME,
+  String componentType, String componentName, String pkg, String version) throws Exception {
+    SolrParams params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME,
+        WT, JAVABIN,
+        "componentName", componentName,
+        "meta", "true"));
+
+    String s = "queryResponseWriter";
+    GenericSolrRequest req1 = new GenericSolrRequest(SolrRequest.METHOD.GET,
+        "/config/" + componentType, params);
+    TestDistribPackageStore.assertResponseValues(10,
+        client,
+        req1, Utils.makeMap(
+            ":config:" + componentType + ":" + componentName + ":_packageinfo_:package", pkg,
+            ":config:" + componentType + ":" + componentName + ":_packageinfo_:version", version
+        ));
+  }
+
+  @Test
+  public void testAPI() throws Exception {
+    System.setProperty("enable.packages", "true");
+    MiniSolrCloudCluster cluster =
+        configureCluster(4)
+            .withJettyConfig(jetty -> jetty.enableV2(true))
+            .addConfig("conf", configset("cloud-minimal"))
+            .configure();
+    try {
+      String errPath = "/error/details[0]/errorMessages[0]";
+      String FILE1 = "/mypkg/v.0.12/jar_a.jar";
+      String FILE2 = "/mypkg/v.0.12/jar_b.jar";
+      String FILE3 = "/mypkg/v.0.13/jar_a.jar";
+
+      PackageAPI.AddVersion add = new PackageAPI.AddVersion();
+      add.version = "0.12";
+      add.pkg = "test_pkg";
+      add.files = Arrays.asList(new String[]{FILE1, FILE2});
+      V2Request req = new V2Request.Builder("/cluster/package")
+          .forceV2(true)
+          .withMethod(SolrRequest.METHOD.POST)
+          .withPayload(Collections.singletonMap("add", add))
+          .build();
+
+      //the files is not yet there. The command should fail with error saying "No such file"
+      expectError(req, cluster.getSolrClient(), errPath, "No such file :");
+
+
+      //post the jar file. No signature is sent
+      postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1, null);
+
+
+      add.files = Arrays.asList(new String[]{FILE1});
+      expectError(req, cluster.getSolrClient(), errPath,
+          FILE1 + " has no signature");
+      //now we upload the keys
+      byte[] derFile = readFile("cryptokeys/pub_key512.der");
+      cluster.getZkClient().makePath("/keys/exe", true);
+      cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
+      //and upload the same file with a different name but it has proper signature
+      postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE2,
+          "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ==");
+      // with correct signature
+      //after uploading the file, let's delete the keys to see if we get proper error message
+      cluster.getZkClient().delete("/keys/exe/pub_key512.der", -1, true);
+      add.files = Arrays.asList(new String[]{FILE2});
+      expectError(req, cluster.getSolrClient(), errPath,
+          "ZooKeeper does not have any public keys");
+
+      //Now lets' put the keys back
+      cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
+
+      //this time we have a file with proper signature, public keys are in ZK
+      // so the add {} command should succeed
+      req.process(cluster.getSolrClient());
+
+      //Now verify the data in ZK
+      TestDistribPackageStore.assertResponseValues(1,
+          () -> new MapWriterMap((Map) Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH,
+              null, new Stat(), true))),
+          Utils.makeMap(
+              ":packages:test_pkg[0]:version", "0.12",
+              ":packages:test_pkg[0]:files[0]", FILE1
+          ));
+
+      //post a new jar with a proper signature
+      postFileAndWait(cluster, "runtimecode/runtimelibs_v2.jar.bin", FILE3,
+          "j+Rflxi64tXdqosIhbusqi6GTwZq8znunC/dzwcWW0/dHlFGKDurOaE1Nz9FSPJuXbHkVLj638yZ0Lp1ssnoYA==");
+
+
+      //this time we are adding the second version of the package (0.13)
+      add.version = "0.13";
+      add.pkg = "test_pkg";
+      add.files = Arrays.asList(new String[]{FILE3});
+
+      //this request should succeed
+      req.process(cluster.getSolrClient());
+      //no verify the data (/packages.json) in ZK
+      TestDistribPackageStore.assertResponseValues(1,
+          () -> new MapWriterMap((Map) Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH,
+              null, new Stat(), true))),
+          Utils.makeMap(
+              ":packages:test_pkg[1]:version", "0.13",
+              ":packages:test_pkg[1]:files[0]", FILE3
+          ));
+
+      //Now we will just delete one version
+      PackageAPI.DelVersion delVersion = new PackageAPI.DelVersion();
+      delVersion.version = "0.1";//this version does not exist
+      delVersion.pkg = "test_pkg";
+      req = new V2Request.Builder("/cluster/package")
+          .forceV2(true)
+          .withMethod(SolrRequest.METHOD.POST)
+          .withPayload(Collections.singletonMap("delete", delVersion))
+          .build();
+
+      //we are expecting an error
+      expectError(req, cluster.getSolrClient(), errPath, "No such version:");
+
+      delVersion.version = "0.12";//correct version. Should succeed
+      req.process(cluster.getSolrClient());
+      //Verify with ZK that the data is correcy
+      TestDistribPackageStore.assertResponseValues(1,
+          () -> new MapWriterMap((Map) Utils.fromJSON(cluster.getZkClient().getData(SOLR_PKGS_PATH,
+              null, new Stat(), true))),
+          Utils.makeMap(
+              ":packages:test_pkg[0]:version", "0.13",
+              ":packages:test_pkg[0]:files[0]", FILE2
+          ));
+
+
+      //So far we have been verifying the details with  ZK directly
+      //use the package read API to verify with each node that it has the correct data
+      for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
+        String path = jetty.getBaseUrl().toString().replace("/solr", "/api") + "/cluster/package?wt=javabin";
+        TestDistribPackageStore.assertResponseValues(10, new Callable<NavigableObject>() {
+          @Override
+          public NavigableObject call() throws Exception {
+            try (HttpSolrClient solrClient = (HttpSolrClient) jetty.newClient()) {
+              return (NavigableObject) Utils.executeGET(solrClient.getHttpClient(), path, Utils.JAVABINCONSUMER);
+            }
+          }
+        }, Utils.makeMap(
+            ":result:packages:test_pkg[0]:version", "0.13",
+            ":result:packages:test_pkg[0]:files[0]", FILE3
+        ));
+      }
+    } finally {
+      cluster.shutdown();
+    }
+  }
+
+  static void postFileAndWait(MiniSolrCloudCluster cluster, String fname, String path, String sig) throws Exception {
+    ByteBuffer fileContent = getFileContent(fname);
+    String sha512 = DigestUtils.sha512Hex(fileContent.array());
+
+    TestDistribPackageStore.postFile(cluster.getSolrClient(),
+        fileContent,
+        path, sig);// has file, but no signature
+
+    TestDistribPackageStore.waitForAllNodesHaveFile(cluster, path, Utils.makeMap(
+        ":files:" + path + ":sha512",
+        sha512
+    ), false);
+  }
+
+  private void expectError(V2Request req, SolrClient client, String errPath, String expectErrorMsg) throws IOException, SolrServerException {
+    try {
+      req.process(client);
+      fail("should have failed with message : " + expectErrorMsg);
+    } catch (BaseHttpSolrClient.RemoteExecutionException e) {
+      String msg = e.getMetaData()._getStr(errPath, "");
+      assertTrue("should have failed with message: " + expectErrorMsg + "actual message : " + msg,
+          msg.contains(expectErrorMsg)
+      );
+    }
+  }
+}
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
index 98f0d9a..b8105ab 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
@@ -34,6 +34,7 @@ import java.nio.file.Path;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.RejectedExecutionException;
+import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
@@ -353,6 +354,10 @@ public class SolrZkClient implements Closeable {
   }
 
   public void atomicUpdate(String path, Function<byte[], byte[]> editor) throws KeeperException, InterruptedException {
+   atomicUpdate(path, (stat, bytes) -> editor.apply(bytes));
+  }
+
+  public void atomicUpdate(String path, BiFunction<Stat , byte[], byte[]> editor) throws KeeperException, InterruptedException {
     for (; ; ) {
       byte[] modified = null;
       byte[] zkData = null;
@@ -360,7 +365,7 @@ public class SolrZkClient implements Closeable {
       try {
         if (exists(path, true)) {
           zkData = getData(path, null, s, true);
-          modified = editor.apply(zkData);
+          modified = editor.apply(s, zkData);
           if (modified == null) {
             //no change , no need to persist
             return;
@@ -368,7 +373,7 @@ public class SolrZkClient implements Closeable {
           setData(path, modified, s.getVersion(), true);
           break;
         } else {
-          modified = editor.apply(null);
+          modified = editor.apply(s,null);
           if (modified == null) {
             //no change , no need to persist
             return;
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index dcf7d9e..5dea5b0 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -118,6 +118,7 @@ public class ZkStateReader implements SolrCloseable {
   public static final String SOLR_AUTOSCALING_TRIGGER_STATE_PATH = "/autoscaling/triggerState";
   public static final String SOLR_AUTOSCALING_NODE_ADDED_PATH = "/autoscaling/nodeAdded";
   public static final String SOLR_AUTOSCALING_NODE_LOST_PATH = "/autoscaling/nodeLost";
+  public static final String SOLR_PKGS_PATH = "/packages.json";
 
   public static final String DEFAULT_SHARD_PREFERENCES = "defaultShardPreferences";
   public static final String REPLICATION_FACTOR = "replicationFactor";


Mime
View raw message