lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From no...@apache.org
Subject lucene-solr:branch_6x: SOLR-9397: Config API does not support adding caches
Date Wed, 10 Aug 2016 14:53:48 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/branch_6x e58c83f0a -> 74b470a7a


SOLR-9397: Config API does not support adding caches


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/74b470a7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/74b470a7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/74b470a7

Branch: refs/heads/branch_6x
Commit: 74b470a7a9516a799ec8f28fac579c461fc8e286
Parents: e58c83f
Author: Noble Paul <noble@apache.org>
Authored: Wed Aug 10 20:23:22 2016 +0530
Committer: Noble Paul <noble@apache.org>
Committed: Wed Aug 10 20:23:22 2016 +0530

----------------------------------------------------------------------
 solr/CHANGES.txt                                |  2 +
 .../java/org/apache/solr/core/SolrConfig.java   | 29 ++++++--
 .../org/apache/solr/search/CacheConfig.java     | 24 ++++---
 .../apache/solr/search/SolrIndexSearcher.java   | 69 +++-----------------
 .../apache/solr/core/TestSolrConfigHandler.java | 56 +++++++++++++++-
 5 files changed, 106 insertions(+), 74 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74b470a7/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index ccbbea1..d47c57e 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -162,6 +162,8 @@ Bug Fixes
 
 * SOLR-9179: NPE in IndexSchema using IBM JDK (noble, Colvin Cowie)
 
+* SOLR-9397: Config API does not support adding caches (noble)
+
 Optimizations
 ----------------------
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74b470a7/solr/core/src/java/org/apache/solr/core/SolrConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
index eb3aa5f..653c612 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
@@ -28,7 +28,17 @@ import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.UUID;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -49,6 +59,7 @@ import org.apache.solr.schema.IndexSchemaFactory;
 import org.apache.solr.search.CacheConfig;
 import org.apache.solr.search.FastLRUCache;
 import org.apache.solr.search.QParserPlugin;
+import org.apache.solr.search.SolrCache;
 import org.apache.solr.search.ValueSourceParser;
 import org.apache.solr.search.stats.StatsCache;
 import org.apache.solr.servlet.SolrRequestParsers;
@@ -91,7 +102,7 @@ public class SolrConfig extends Config implements MapSerializable {
   public static final String DEFAULT_CONF_FILE = "solrconfig.xml";
   private RequestParams requestParams;
 
-  public static enum PluginOpts {
+  public enum PluginOpts {
     MULTI_OK,
     REQUIRE_NAME,
     REQUIRE_NAME_IN_OVERLAY,
@@ -254,7 +265,6 @@ public class SolrConfig extends Config implements MapSerializable {
     dataDir = get("dataDir", null);
     if (dataDir != null && dataDir.length() == 0) dataDir = null;
 
-    userCacheConfigs = CacheConfig.getMultipleConfigs(this, "query/cache");
 
     org.apache.solr.search.SolrIndexSearcher.initRegenerators(this);
 
@@ -276,6 +286,16 @@ public class SolrConfig extends Config implements MapSerializable {
     maxWarmingSearchers = getInt("query/maxWarmingSearchers", Integer.MAX_VALUE);
     slowQueryThresholdMillis = getInt("query/slowQueryThresholdMillis", -1);
     for (SolrPluginInfo plugin : plugins) loadPluginInfo(plugin);
+
+    Map<String, CacheConfig> userCacheConfigs = CacheConfig.getMultipleConfigs(this,
"query/cache");
+    List<PluginInfo> caches = getPluginInfos(SolrCache.class.getName());
+    if (!caches.isEmpty()) {
+      for (PluginInfo c : caches) {
+        userCacheConfigs.put(c.name, CacheConfig.getConfig(this, "cache", c.attributes, null));
+      }
+    }
+    this.userCacheConfigs = Collections.unmodifiableMap(userCacheConfigs);
+
     updateHandlerInfo = loadUpdatehandlerInfo();
 
     multipartUploadLimitKB = getInt(
@@ -317,6 +337,7 @@ public class SolrConfig extends Config implements MapSerializable {
       .add(new SolrPluginInfo(TransformerFactory.class, "transformer", REQUIRE_NAME, REQUIRE_CLASS,
MULTI_OK))
       .add(new SolrPluginInfo(SearchComponent.class, "searchComponent", REQUIRE_NAME, REQUIRE_CLASS,
MULTI_OK))
       .add(new SolrPluginInfo(UpdateRequestProcessorFactory.class, "updateProcessor", REQUIRE_NAME,
REQUIRE_CLASS, MULTI_OK))
+      .add(new SolrPluginInfo(SolrCache.class, "cache", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK))
           // TODO: WTF is up with queryConverter???
           // it apparently *only* works as a singleton? - SOLR-4304
           // and even then -- only if there is a single SpellCheckComponent
@@ -457,7 +478,7 @@ public class SolrConfig extends Config implements MapSerializable {
   public final CacheConfig queryResultCacheConfig;
   public final CacheConfig documentCacheConfig;
   public final CacheConfig fieldValueCacheConfig;
-  public final CacheConfig[] userCacheConfigs;
+  public final Map<String, CacheConfig> userCacheConfigs;
   // SolrIndexSearcher - more...
   public final boolean useFilterForSortedQuery;
   public final int queryResultWindowSize;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74b470a7/solr/core/src/java/org/apache/solr/search/CacheConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/CacheConfig.java b/solr/core/src/java/org/apache/solr/search/CacheConfig.java
index 40e54dc..ee333f8 100644
--- a/solr/core/src/java/org/apache/solr/search/CacheConfig.java
+++ b/solr/core/src/java/org/apache/solr/search/CacheConfig.java
@@ -17,10 +17,10 @@
 package org.apache.solr.search;
 
 import javax.xml.xpath.XPathConstants;
-
 import java.lang.invoke.MethodHandles;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -75,14 +75,15 @@ public class CacheConfig implements MapSerializable{
     this.regenerator = regenerator;
   }
 
-  public static CacheConfig[] getMultipleConfigs(SolrConfig solrConfig, String configPath)
{
-    NodeList nodes = (NodeList)solrConfig.evaluate(configPath, XPathConstants.NODESET);
-    if (nodes==null || nodes.getLength()==0) return null;
-    CacheConfig[] configs = new CacheConfig[nodes.getLength()];
-    for (int i=0; i<nodes.getLength(); i++) {
-      configs[i] = getConfig(solrConfig, nodes.item(i).getNodeName(), DOMUtil.toMap(nodes.item(i).getAttributes()),
configPath);
+  public static Map<String, CacheConfig> getMultipleConfigs(SolrConfig solrConfig,
String configPath) {
+    NodeList nodes = (NodeList) solrConfig.evaluate(configPath, XPathConstants.NODESET);
+    if (nodes == null || nodes.getLength() == 0) return new LinkedHashMap<>();
+    Map<String, CacheConfig> result = new HashMap<>(nodes.getLength());
+    for (int i = 0; i < nodes.getLength(); i++) {
+      CacheConfig config = getConfig(solrConfig, nodes.item(i).getNodeName(), DOMUtil.toMap(nodes.item(i).getAttributes()),
configPath);
+      result.put(config.args.get(NAME), config);
     }
-    return configs;
+    return result;
   }
 
 
@@ -101,9 +102,14 @@ public class CacheConfig implements MapSerializable{
   public static CacheConfig getConfig(SolrConfig solrConfig, String nodeName, Map<String,String>
attrs, String xpath) {
     CacheConfig config = new CacheConfig();
     config.nodeName = nodeName;
+    Map attrsCopy = new LinkedHashMap<>(attrs.size());
+    for (Map.Entry<String, String> e : attrs.entrySet()) {
+      attrsCopy.put(e.getKey(), String.valueOf(e.getValue()));
+    }
+    attrs = attrsCopy;
     config.args = attrs;
 
-    Map<String, String> map = solrConfig.getOverlay().getEditableSubProperties(xpath);
+    Map<String, String> map = xpath == null ? null : solrConfig.getOverlay().getEditableSubProperties(xpath);
     if(map != null){
       HashMap<String, String> mapCopy = new HashMap<>(config.args);
       for (Map.Entry<String, String> e : map.entrySet()) {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74b470a7/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index cc719f0..0f480c6 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -36,62 +36,16 @@ import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
+import com.google.common.base.Function;
+import com.google.common.base.Objects;
+import com.google.common.collect.Iterables;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.DocumentStoredFieldVisitor;
 import org.apache.lucene.document.LazyDocument;
-import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.DocValuesType;
-import org.apache.lucene.index.ExitableDirectoryReader;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.MultiPostingsEnum;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.*;
 import org.apache.lucene.index.StoredFieldVisitor.Status;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermContext;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.*;
 import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.CollectionStatistics;
-import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.DocIdSet;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.EarlyTerminatingSortingCollector;
-import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.FieldDoc;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.LeafCollector;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.MultiCollector;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.SimpleCollector;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TermStatistics;
-import org.apache.lucene.search.TimeLimitingCollector;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.TopDocsCollector;
-import org.apache.lucene.search.TopFieldCollector;
-import org.apache.lucene.search.TopFieldDocs;
-import org.apache.lucene.search.TopScoreDocCollector;
-import org.apache.lucene.search.TotalHitCountCollector;
-import org.apache.lucene.search.Weight;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
@@ -128,10 +82,6 @@ import org.apache.solr.update.SolrIndexConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Function;
-import com.google.common.base.Objects;
-import com.google.common.collect.Iterables;
-
 /**
  * SolrIndexSearcher adds schema awareness and caching functionality over {@link IndexSearcher}.
  *
@@ -337,13 +287,12 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,
SolrI
       documentCache = solrConfig.documentCacheConfig == null ? null : solrConfig.documentCacheConfig.newInstance();
       if (documentCache != null) clist.add(documentCache);
 
-      if (solrConfig.userCacheConfigs == null) {
+      if (solrConfig.userCacheConfigs.isEmpty()) {
         cacheMap = NO_GENERIC_CACHES;
       } else {
-        cacheMap = new HashMap<>(solrConfig.userCacheConfigs.length);
-        for (CacheConfig userCacheConfig : solrConfig.userCacheConfigs) {
-          SolrCache cache = null;
-          if (userCacheConfig != null) cache = userCacheConfig.newInstance();
+        cacheMap = new HashMap<>(solrConfig.userCacheConfigs.size());
+        for (Map.Entry<String,CacheConfig> e : solrConfig.userCacheConfigs.entrySet())
{
+          SolrCache cache = e.getValue().newInstance();
           if (cache != null) {
             cacheMap.put(cache.name(), cache);
             clist.add(cache);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74b470a7/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
index 250ccf0..9b71ff8 100644
--- a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
+++ b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
@@ -22,6 +22,7 @@ import java.io.StringReader;
 import java.lang.invoke.MethodHandles;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -35,8 +36,12 @@ import org.apache.solr.SolrTestCaseJ4;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.common.util.StrUtils;
 import org.apache.solr.common.util.Utils;
+import org.apache.solr.handler.DumpRequestHandler;
 import org.apache.solr.handler.TestBlobHandler;
 import org.apache.solr.handler.TestSolrConfigHandlerConcurrent;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.search.SolrCache;
 import org.apache.solr.util.RestTestBase;
 import org.apache.solr.util.RestTestHarness;
 import org.eclipse.jetty.servlet.ServletHolder;
@@ -449,7 +454,56 @@ public class TestSolrConfigHandler extends RestTestBase {
     assertEquals(2, initArgs.size());
     assertTrue(((Map)initArgs.get(0)).containsKey("suggester"));
     assertTrue(((Map)initArgs.get(1)).containsKey("suggester"));
-    System.out.println(map);
+
+    payload = "{\n" +
+        "'add-requesthandler' : { 'name' : '/dump101', 'class': " +
+        "'" + CacheTest.class.getName() + "' " +
+        ", 'startup' : 'lazy'}\n" +
+        "}";
+    runConfigCommand(writeHarness, "/config?wt=json", payload);
+
+    testForResponseElement(writeHarness,
+        testServerBaseUrl,
+        "/config/overlay?wt=json",
+        cloudSolrClient,
+        Arrays.asList("overlay", "requestHandler", "/dump101", "startup"),
+        "lazy",
+        10);
+
+    payload = "{\n" +
+        "'add-cache' : {name:'lfuCacheDecayFalse', class:'solr.search.LFUCache', size:10
,initialSize:9 , timeDecay:false }," +
+        "'add-cache' : {name: 'perSegFilter', class: 'solr.search.LRUCache', size:10, initialSize:0
, autowarmCount:10}}";
+    runConfigCommand(writeHarness, "/config?wt=json", payload);
+
+    map = testForResponseElement(writeHarness,
+        testServerBaseUrl,
+        "/config/overlay?wt=json",
+        cloudSolrClient,
+        Arrays.asList("overlay", "cache", "lfuCacheDecayFalse", "class"),
+        "solr.search.LFUCache",
+        10);
+    assertEquals("solr.search.LRUCache",getObjectByPath(map, true, ImmutableList.of("overlay",
"cache", "perSegFilter", "class")));
+
+    map = getRespMap("/dump101?cacheNames=lfuCacheDecayFalse&cacheNames=perSegFilter&wt=json",
writeHarness);
+    assertEquals("Actual output "+ Utils.toJSONString(map), "org.apache.solr.search.LRUCache",getObjectByPath(map,
true, ImmutableList.of( "caches", "perSegFilter")));
+    assertEquals("Actual output "+ Utils.toJSONString(map), "org.apache.solr.search.LFUCache",getObjectByPath(map,
true, ImmutableList.of( "caches", "lfuCacheDecayFalse")));
+
+  }
+
+  public static class CacheTest extends DumpRequestHandler {
+    @Override
+    public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException
{
+      super.handleRequestBody(req, rsp);
+      String[] caches = req.getParams().getParams("cacheNames");
+      if(caches != null && caches.length>0){
+        HashMap m = new HashMap();
+        rsp.add("caches", m);
+        for (String c : caches) {
+          SolrCache cache = req.getSearcher().getCache(c);
+          if(cache != null) m.put(c, cache.getClass().getName());
+        }
+      }
+    }
   }
 
   public static Map testForResponseElement(RestTestHarness harness,


Mime
View raw message