lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject [lucene-solr] branch jira/solr-13579 updated: SOLR-13579: SolrIndexSearcher caches are now split into separate pools by function.
Date Thu, 09 Jan 2020 17:55:10 GMT
This is an automated email from the ASF dual-hosted git repository.

ab pushed a commit to branch jira/solr-13579
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git


The following commit(s) were added to refs/heads/jira/solr-13579 by this push:
     new ba6c42f  SOLR-13579: SolrIndexSearcher caches are now split into separate pools by
function.
ba6c42f is described below

commit ba6c42f9c47d6073b5125dfb0001846feecd22db
Author: Andrzej Bialecki <ab@apache.org>
AuthorDate: Thu Jan 9 18:54:11 2020 +0100

    SOLR-13579: SolrIndexSearcher caches are now split into separate pools by function.
---
 .../solr/handler/admin/ResourceManagerHandler.java | 111 ++++++++++++---------
 .../solr/managed/DefaultResourceManager.java       |  27 +++--
 .../org/apache/solr/managed/ResourceManager.java   |   8 +-
 .../java/org/apache/solr/search/CaffeineCache.java |   6 ++
 .../org/apache/solr/search/SolrIndexSearcher.java  |  44 ++++++--
 5 files changed, 132 insertions(+), 64 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ResourceManagerHandler.java
b/solr/core/src/java/org/apache/solr/handler/admin/ResourceManagerHandler.java
index 72da514..13a9136 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/ResourceManagerHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/ResourceManagerHandler.java
@@ -18,10 +18,12 @@ package org.apache.solr.handler.admin;
 
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.stream.Collectors;
@@ -124,47 +126,56 @@ public class ResourceManagerHandler extends RequestHandlerBase implements
Permis
     if (op == null) {
       throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unsupported pool operation:
" + params.get(POOL_ACTION_PARAM));
     }
-    String name = null;
-    if (op != PoolOp.LIST) {
-      name = params.get(CommonParams.NAME);
+    final String name = params.get(CommonParams.NAME);
+    if (op == PoolOp.CREATE) {
       if (name == null) {
         throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Required parameter
" + CommonParams.NAME + " missing: " + params);
       }
     }
     NamedList<Object> result = new SimpleOrderedMap<>();
+    Set<String> poolNames = new TreeSet<String>(
+        name != null ?
+            resourceManager.listPools().stream()
+                .filter(n -> n.startsWith(name))
+                .collect(Collectors.toSet())
+        :
+        resourceManager.listPools());
     switch (op) {
       case LIST:
-        resourceManager.listPools().forEach(p -> {
+        poolNames.forEach(p -> {
           ResourceManagerPool pool = resourceManager.getPool(p);
           if (pool == null) {
             return;
           }
-          NamedList<Object> perPool = new SimpleOrderedMap<>();
+          SimpleOrderedMap<Object> perPool = new SimpleOrderedMap<>();
           result.add(p, perPool);
           perPool.add("type", pool.getType());
           perPool.add("size", pool.getComponents().size());
           perPool.add("poolLimits", pool.getPoolLimits());
           perPool.add("poolParams", pool.getParams());
-          perPool.add("resources", new TreeSet<>(pool.getComponents().keySet()));
         });
         break;
       case STATUS:
-        ResourceManagerPool pool = resourceManager.getPool(name);
-        if (pool == null) {
-          throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "Pool '" + name + "'
not found.");
-        }
-        result.add("type", pool.getType());
-        result.add("size", pool.getComponents().size());
-        result.add("poolLimits", pool.getPoolLimits());
-        result.add("poolParams", pool.getParams());
-        result.add("resources", new TreeSet<>(pool.getComponents().keySet()));
-        try {
-          Map<String, Map<String, Object>> values = pool.getCurrentValues();
-          result.add("totalValues", new TreeMap<>(pool.aggregateTotalValues(values)));
-        } catch (Exception e) {
-          log.warn("Error getting current values from pool " + name, e);
-          result.add("error", "Error getting current values: " + e.toString());
-        }
+        poolNames.forEach(p -> {
+          ResourceManagerPool pool = resourceManager.getPool(p);
+          if (pool == null) {
+            return;
+          }
+          SimpleOrderedMap<Object> perPool = new SimpleOrderedMap<>();
+          result.add(p, perPool);
+          perPool.add("type", pool.getType());
+          perPool.add("size", pool.getComponents().size());
+          perPool.add("poolLimits", pool.getPoolLimits());
+          perPool.add("poolParams", pool.getParams());
+          perPool.add("resources", new TreeSet<>(pool.getComponents().keySet()));
+          try {
+            Map<String, Map<String, Object>> values = pool.getCurrentValues();
+            perPool.add("totalValues", new TreeMap<>(pool.aggregateTotalValues(values)));
+          } catch (Exception e) {
+            log.warn("Error getting current values from pool " + name, e);
+            perPool.add("error", "Error getting current values: " + e.toString());
+          }
+        });
         break;
       case CREATE:
         String type = params.get(CommonParams.TYPE);
@@ -181,29 +192,33 @@ public class ResourceManagerHandler extends RequestHandlerBase implements
Permis
         }
         break;
       case DELETE:
-        try {
-          resourceManager.removePool(name);
-          result.add("success", "removed");
-        } catch (Exception e) {
-          throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Pool '" + name +
"' deletion failed: " + e.toString(), e);
-        }
+        poolNames.forEach(p -> {
+          try {
+            resourceManager.removePool(p);
+            result.add(p, "success");
+          } catch (Exception e) {
+            result.add(p, "error: " + e.toString());
+          }
+        });
         break;
       case SETLIMITS:
-        ResourceManagerPool pool1 = resourceManager.getPool(name);
-        if (pool1 == null) {
-          throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "Pool '" + name + "'
not found.");
-        }
-        Map<String, Object> currentLimits = new HashMap<>(pool1.getPoolLimits());
-        Map<String, Object> newLimits = getMap(params, LIMIT_PREFIX_PARAM);
-        newLimits.forEach((k, v) -> {
-          if (v == null) {
-            currentLimits.remove(k);
-          } else {
-            currentLimits.put(k, v);
+        poolNames.forEach(p -> {
+          ResourceManagerPool pool = resourceManager.getPool(p);
+          if (pool == null) {
+            return;
           }
+          Map<String, Object> currentLimits = new HashMap<>(pool.getPoolLimits());
+          Map<String, Object> newLimits = getMap(params, LIMIT_PREFIX_PARAM);
+          newLimits.forEach((k, v) -> {
+            if (v == null) {
+              currentLimits.remove(k);
+            } else {
+              currentLimits.put(k, v);
+            }
+          });
+          pool.setPoolLimits(newLimits);
+          result.add(p, newLimits);
         });
-        pool1.setPoolLimits(newLimits);
-        result.add("success", newLimits);
         break;
     }
     rsp.getValues().add("result", result);
@@ -250,18 +265,24 @@ public class ResourceManagerHandler extends RequestHandlerBase implements
Permis
       throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "Pool '" + poolName + "'
not found.");
     }
     String resName = params.get(CommonParams.NAME);
-    if ((resName == null || resName.isBlank()) && op != ResOp.LIST) {
-      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Missing '" + CommonParams.NAME
+ "' parameter.");
-    }
     NamedList<Object> result = new SimpleOrderedMap<>();
+    // we support a prefix of resource names because eg. searcher caches will have a quickly
+    // changing unique suffix
     List<ManagedComponent> components = resName == null ? new ArrayList<>(pool.getComponents().values())
: pool.getComponents().values().stream()
+        .sorted(Comparator.comparing(c -> c.getManagedComponentId().toString()))
         .filter(c -> c.getManagedComponentId().toString().startsWith(resName)).collect(Collectors.toList());
     if (op != ResOp.LIST && components.isEmpty()) {
       throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "Component(s) '" + resName
+ " not found in pool '" + poolName + "'.");
     }
     switch (op) {
       case LIST:
-        pool.getComponents().forEach((n, component) -> {
+        Set<String> componentNames = new TreeSet<>(pool.getComponents().keySet());
+        componentNames.forEach(n -> {
+          ManagedComponent component = pool.getComponents().get(n);
+          if (component == null) {
+            // removed in the meantime
+            return;
+          }
           NamedList<Object> perRes = new SimpleOrderedMap<>();
           result.add(n, perRes);
           perRes.add("class", component.getClass().getName());
diff --git a/solr/core/src/java/org/apache/solr/managed/DefaultResourceManager.java b/solr/core/src/java/org/apache/solr/managed/DefaultResourceManager.java
index 491581f..4f540b0 100644
--- a/solr/core/src/java/org/apache/solr/managed/DefaultResourceManager.java
+++ b/solr/core/src/java/org/apache/solr/managed/DefaultResourceManager.java
@@ -52,10 +52,14 @@ public class DefaultResourceManager extends ResourceManager {
   public static final String SCHEDULE_DELAY_SECONDS_PARAM = "scheduleDelaySeconds";
   public static final String MAX_NUM_POOLS_PARAM = "maxNumPools";
 
-  public static final int DEFAULT_MAX_POOLS = 100;
+  public static final int DEFAULT_MAX_POOLS = 200;
   public static final int DEFAULT_SCHEDULE_DELAY_SECONDS = 10;
 
-  public static final String NODE_SEARCHER_CACHE_POOL = "nodeSearcherCachePool";
+  public static final String DOCUMENT_CACHE_POOL = "searcherDocumentCache";
+  public static final String FILTER_CACHE_POOL = "searcherFilterCache";
+  public static final String FIELD_VALUE_CACHE_POOL = "searcherFieldValueCache";
+  public static final String QUERY_RESULT_CACHE_POOL = "searcherQueryResultCache";
+  public static final String USER_SEARCHER_CACHE_POOL = "searcherUserCache";
 
   public static final Map<String, Map<String, Object>> DEFAULT_NODE_POOLS = new
HashMap<>();
 
@@ -64,7 +68,13 @@ public class DefaultResourceManager extends ResourceManager {
     params.put(CommonParams.TYPE, CacheManagerPool.TYPE);
     // unlimited RAM
     params.put(SolrCache.MAX_RAM_MB_PARAM, -1L);
-    DEFAULT_NODE_POOLS.put(NODE_SEARCHER_CACHE_POOL, params);
+    // unlimited size
+    params.put(SolrCache.MAX_SIZE_PARAM, -1L);
+    DEFAULT_NODE_POOLS.put(DOCUMENT_CACHE_POOL, new HashMap<>(params));
+    DEFAULT_NODE_POOLS.put(FILTER_CACHE_POOL, new HashMap<>(params));
+    DEFAULT_NODE_POOLS.put(FIELD_VALUE_CACHE_POOL, new HashMap<>(params));
+    DEFAULT_NODE_POOLS.put(QUERY_RESULT_CACHE_POOL, new HashMap<>(params));
+    DEFAULT_NODE_POOLS.put(USER_SEARCHER_CACHE_POOL, new HashMap<>(params));
   }
 
 
@@ -95,7 +105,7 @@ public class DefaultResourceManager extends ResourceManager {
   }
 
   protected void doInit() throws Exception {
-    scheduledThreadPoolExecutor = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(2,
+    scheduledThreadPoolExecutor = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(DEFAULT_NODE_POOLS.size(),
         new DefaultSolrThreadFactory(getClass().getSimpleName()));
     scheduledThreadPoolExecutor.setMaximumPoolSize(maxNumPools);
     scheduledThreadPoolExecutor.setRemoveOnCancelPolicy(true);
@@ -183,7 +193,8 @@ public class DefaultResourceManager extends ResourceManager {
     ensureActive();
     ResourceManagerPool pool = resourcePools.remove(name);
     if (pool == null) {
-      throw new IllegalArgumentException("Pool '" + name + "' doesn't exist.");
+      log.warn("Pool '" + name + "' doesn't exist, ignoring remove request.");
+      return;
     }
     IOUtils.closeQuietly(pool);
     log.info("- removed pool " + pool.getName() + " / " + pool.getType());
@@ -194,7 +205,8 @@ public class DefaultResourceManager extends ResourceManager {
     ensureActive();
     ResourceManagerPool pool = resourcePools.get(name);
     if (pool == null) {
-      throw new IllegalArgumentException("Pool '" + name + "' doesn't exist.");
+      log.warn("Pool '" + name + "' doesn't exist, ignoring register of " + managedComponent.getManagedComponentId());
+      return;
     }
     String poolType = pool.getType();
     resourcePools.forEach((poolName, otherPool) -> {
@@ -216,7 +228,8 @@ public class DefaultResourceManager extends ResourceManager {
     ensureActive();
     ResourceManagerPool pool = resourcePools.get(poolName);
     if (pool == null) {
-      throw new IllegalArgumentException("Pool '" + poolName + "' doesn't exist.");
+      log.warn("Pool '" + poolName + "' doesn't exist, ignoring unregister of " + resourceId);
+      return false;
     }
     return pool.unregisterComponent(resourceId);
   }
diff --git a/solr/core/src/java/org/apache/solr/managed/ResourceManager.java b/solr/core/src/java/org/apache/solr/managed/ResourceManager.java
index 3d8a733..4ed2736 100644
--- a/solr/core/src/java/org/apache/solr/managed/ResourceManager.java
+++ b/solr/core/src/java/org/apache/solr/managed/ResourceManager.java
@@ -17,16 +17,13 @@
 
 package org.apache.solr.managed;
 
-import java.io.Closeable;
 import java.lang.invoke.MethodHandles;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
-import org.apache.solr.common.SolrCloseable;
 import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.util.IOUtils;
 import org.apache.solr.common.util.TimeSource;
 import org.apache.solr.core.PluginInfo;
 import org.apache.solr.core.SolrResourceLoader;
@@ -198,9 +195,12 @@ public abstract class ResourceManager implements PluginInfoInitialized,
SolrMetr
    */
   public abstract Collection<String> listPools();
 
-  /** Get a named pool. */
+  /** Return a named pool or null if no such pool exists. */
   public abstract ResourceManagerPool getPool(String name);
 
+  public boolean hasPool(String name) {
+    return getPool(name) != null;
+  }
   /**
    * Modify pool limits of an existing pool.
    * @param name existing pool name
diff --git a/solr/core/src/java/org/apache/solr/search/CaffeineCache.java b/solr/core/src/java/org/apache/solr/search/CaffeineCache.java
index 717e9d2..aa4c9ef 100644
--- a/solr/core/src/java/org/apache/solr/search/CaffeineCache.java
+++ b/solr/core/src/java/org/apache/solr/search/CaffeineCache.java
@@ -296,6 +296,12 @@ public class CaffeineCache<K, V> extends SolrCacheBase implements
SolrCache<K, V
     if (regenerator == null) {
       return;
     }
+
+    // inherit also maxSize / maxRamMB from the old cache - these may
+    // be currently set to different values than the static config due to
+    // dynamic adjustments
+    setMaxSize(old.getMaxSize());
+    setMaxRamMB(old.getMaxRamMB());
     
     long warmingStartTime = System.nanoTime();
     Map<K, V> hottest = Collections.emptyMap();
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index 3994b2c..48497f7 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -59,6 +59,7 @@ import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.util.ObjectReleaseTracker;
+import org.apache.solr.common.util.Utils;
 import org.apache.solr.core.DirectoryFactory;
 import org.apache.solr.core.DirectoryFactory.DirContext;
 import org.apache.solr.core.SolrConfig;
@@ -126,6 +127,8 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,
SolrI
   // map of generic caches - not synchronized since it's read-only after the constructor.
   private final Map<String,SolrCache> cacheMap;
 
+  private final Map<String, List<SolrCache>> cacheByPool;
+
   // list of all caches associated with this searcher.
   private final SolrCache[] cacheList;
 
@@ -272,16 +275,33 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,
SolrI
     this.cachingEnabled = enableCache;
     if (cachingEnabled) {
       final ArrayList<SolrCache> clist = new ArrayList<>();
+      cacheByPool = new HashMap<>();
       fieldValueCache = solrConfig.fieldValueCacheConfig == null ? null
           : solrConfig.fieldValueCacheConfig.newInstance();
-      if (fieldValueCache != null) clist.add(fieldValueCache);
+      if (fieldValueCache != null) {
+        clist.add(fieldValueCache);
+        cacheByPool.computeIfAbsent(DefaultResourceManager.FIELD_VALUE_CACHE_POOL, Utils.NEW_ARRAYLIST_FUN)
+            .add(fieldValueCache);
+      }
       filterCache = solrConfig.filterCacheConfig == null ? null : solrConfig.filterCacheConfig.newInstance();
-      if (filterCache != null) clist.add(filterCache);
+      if (filterCache != null) {
+        clist.add(filterCache);
+        cacheByPool.computeIfAbsent(DefaultResourceManager.FILTER_CACHE_POOL, Utils.NEW_ARRAYLIST_FUN)
+            .add(filterCache);
+      }
       queryResultCache = solrConfig.queryResultCacheConfig == null ? null
           : solrConfig.queryResultCacheConfig.newInstance();
-      if (queryResultCache != null) clist.add(queryResultCache);
+      if (queryResultCache != null) {
+        clist.add(queryResultCache);
+        cacheByPool.computeIfAbsent(DefaultResourceManager.QUERY_RESULT_CACHE_POOL, Utils.NEW_ARRAYLIST_FUN)
+            .add(queryResultCache);
+      }
       SolrCache<Integer, Document> documentCache = docFetcher.getDocumentCache();
-      if (documentCache != null) clist.add(documentCache);
+      if (documentCache != null) {
+        clist.add(documentCache);
+        cacheByPool.computeIfAbsent(DefaultResourceManager.DOCUMENT_CACHE_POOL, Utils.NEW_ARRAYLIST_FUN)
+            .add(documentCache);
+      }
 
       if (solrConfig.userCacheConfigs.isEmpty()) {
         cacheMap = NO_GENERIC_CACHES;
@@ -292,6 +312,8 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,
SolrI
           if (cache != null) {
             cacheMap.put(cache.name(), cache);
             clist.add(cache);
+            cacheByPool.computeIfAbsent(DefaultResourceManager.USER_SEARCHER_CACHE_POOL,
Utils.NEW_ARRAYLIST_FUN)
+                .add(cache);
           }
         }
       }
@@ -303,6 +325,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,
SolrI
       this.fieldValueCache = null;
       this.cacheMap = NO_GENERIC_CACHES;
       this.cacheList = NO_CACHES;
+      this.cacheByPool = Collections.emptyMap();
     }
 
     // We already have our own filter cache
@@ -436,10 +459,15 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,
SolrI
     this.solrMetricsContext = core.getSolrMetricsContext().getChildContext(this, null);
     for (SolrCache cache : cacheList) {
       cache.initializeMetrics(solrMetricsContext, SolrMetricManager.mkName(cache.name(),
STATISTICS_KEY));
-      try {
-        cache.initializeManagedComponent(resourceManager, DefaultResourceManager.NODE_SEARCHER_CACHE_POOL);
-      } catch (Exception e) {
-        log.warn("Exception adding cache '" + cache.getManagedComponentId() + "' to the resource
manager pool", e);
+    }
+    // register caches in their respective resource pools
+    for (Map.Entry<String, List<SolrCache>> entry : cacheByPool.entrySet()) {
+      for (SolrCache cache : entry.getValue()) {
+        try {
+          cache.initializeManagedComponent(resourceManager, entry.getKey());
+        } catch (Exception e) {
+          log.warn("Exception adding cache '" + cache.getManagedComponentId() + "' to the
resource manager pool " + entry.getKey(), e);
+        }
       }
     }
     initializeMetrics(solrMetricsContext, STATISTICS_KEY);


Mime
View raw message