lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject [50/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-11779: Add unit tests.
Date Tue, 08 May 2018 21:15:36 GMT
SOLR-11779: Add unit tests.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b5fa6653
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b5fa6653
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b5fa6653

Branch: refs/heads/jira/solr-11779
Commit: b5fa665387f6d7d8ab079f4cbe703e21f3465840
Parents: c333759
Author: Andrzej Bialecki <ab@apache.org>
Authored: Tue May 8 23:14:22 2018 +0200
Committer: Andrzej Bialecki <ab@apache.org>
Committed: Tue May 8 23:14:22 2018 +0200

----------------------------------------------------------------------
 .../org/apache/solr/core/CoreContainer.java     |  23 +-
 .../solr/handler/admin/MetricsHandler.java      |  19 +-
 .../handler/admin/MetricsHistoryHandler.java    |  71 +++++-
 .../apache/solr/metrics/rrd/SolrRrdBackend.java |   4 +
 .../solr/metrics/rrd/SolrRrdBackendFactory.java |  58 ++++-
 .../solr/handler/admin/MetricsHandlerTest.java  |   8 +-
 .../admin/MetricsHistoryHandlerTest.java        |  26 ++
 .../metrics/rrd/SolrRrdBackendFactoryTest.java  | 244 +++++++++++++++++++
 .../apache/solr/common/params/CommonParams.java |   1 +
 9 files changed, 421 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b5fa6653/solr/core/src/java/org/apache/solr/core/CoreContainer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 9caadd2..b40fc32 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -25,6 +25,7 @@ import static org.apache.solr.common.params.CommonParams.CONFIGSETS_HANDLER_PATH
 import static org.apache.solr.common.params.CommonParams.CORES_HANDLER_PATH;
 import static org.apache.solr.common.params.CommonParams.HEALTH_CHECK_HANDLER_PATH;
 import static org.apache.solr.common.params.CommonParams.INFO_HANDLER_PATH;
+import static org.apache.solr.common.params.CommonParams.METRICS_HISTORY_PATH;
 import static org.apache.solr.common.params.CommonParams.METRICS_PATH;
 import static org.apache.solr.common.params.CommonParams.ZK_PATH;
 import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;
@@ -56,6 +57,7 @@ import org.apache.http.client.CredentialsProvider;
 import org.apache.http.config.Lookup;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.store.Directory;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
 import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
 import org.apache.solr.client.solrj.impl.SolrHttpClientContextBuilder;
@@ -87,6 +89,7 @@ import org.apache.solr.handler.admin.HealthCheckHandler;
 import org.apache.solr.handler.admin.InfoHandler;
 import org.apache.solr.handler.admin.MetricsCollectorHandler;
 import org.apache.solr.handler.admin.MetricsHandler;
+import org.apache.solr.handler.admin.MetricsHistoryHandler;
 import org.apache.solr.handler.admin.SecurityConfHandler;
 import org.apache.solr.handler.admin.SecurityConfHandlerLocal;
 import org.apache.solr.handler.admin.SecurityConfHandlerZk;
@@ -190,6 +193,8 @@ public class CoreContainer {
 
   protected MetricsHandler metricsHandler;
 
+  protected MetricsHistoryHandler metricsHistoryHandler;
+
   protected MetricsCollectorHandler metricsCollectorHandler;
 
   protected AutoscalingHistoryHandler autoscalingHistoryHandler;
@@ -540,7 +545,20 @@ public class CoreContainer {
     infoHandler        = createHandler(INFO_HANDLER_PATH, cfg.getInfoHandlerClass(), InfoHandler.class);
     coreAdminHandler   = createHandler(CORES_HANDLER_PATH, cfg.getCoreAdminHandlerClass(),
CoreAdminHandler.class);
     configSetsHandler = createHandler(CONFIGSETS_HANDLER_PATH, cfg.getConfigSetsHandlerClass(),
ConfigSetsHandler.class);
-    metricsHandler = createHandler(METRICS_PATH, MetricsHandler.class.getName(), MetricsHandler.class);
+
+    // metricsHistoryHandler uses metricsHandler, so create it first
+    metricsHandler = new MetricsHandler(metricManager);
+    containerHandlers.put(METRICS_PATH, metricsHandler);
+    metricsHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag,
METRICS_PATH);
+
+    if (isZooKeeperAware()) {
+      metricsHistoryHandler = new MetricsHistoryHandler(metricManager, metricsHandler,
+          new CloudSolrClient.Builder(Collections.singletonList(getZkController().getZkServerAddress()),
Optional.empty())
+      .withHttpClient(updateShardHandler.getDefaultHttpClient()).build(), getZkController().getSolrCloudManager());
+      containerHandlers.put(METRICS_HISTORY_PATH, metricsHistoryHandler);
+      metricsHistoryHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(),
metricTag, METRICS_HISTORY_PATH);
+    }
+
     autoscalingHistoryHandler = createHandler(AUTOSCALING_HISTORY_PATH, AutoscalingHistoryHandler.class.getName(),
AutoscalingHistoryHandler.class);
     metricsCollectorHandler = createHandler(MetricsCollectorHandler.HANDLER_PATH, MetricsCollectorHandler.class.getName(),
MetricsCollectorHandler.class);
     // may want to add some configuration here in the future
@@ -765,6 +783,9 @@ public class CoreContainer {
       } catch (Exception e) {
         log.warn("Error removing live node. Continuing to close CoreContainer", e);
       }
+      if (metricsHistoryHandler != null) {
+        IOUtils.closeQuietly(metricsHistoryHandler.getSolrClient());
+      }
       if (metricManager != null) {
         metricManager.closeReporters(SolrMetricManager.getRegistryName(SolrInfoBean.Group.cluster));
       }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b5fa6653/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
index 3c8a152..6f1c5e0 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
@@ -54,7 +54,6 @@ import org.apache.solr.util.stats.MetricUtils;
  * Request handler to return metrics
  */
 public class MetricsHandler extends RequestHandlerBase implements PermissionNameProvider
{
-  final CoreContainer container;
   final SolrMetricManager metricManager;
 
   public static final String COMPACT_PARAM = "compact";
@@ -71,13 +70,11 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
   private static final Pattern KEY_REGEX = Pattern.compile("(?<!" + Pattern.quote("\\")
+ ")" + Pattern.quote(":"));
 
   public MetricsHandler() {
-    this.container = null;
     this.metricManager = null;
   }
 
-  public MetricsHandler(CoreContainer container) {
-    this.container = container;
-    this.metricManager = this.container.getMetricManager();
+  public MetricsHandler(SolrMetricManager metricManager) {
+    this.metricManager = metricManager;
   }
 
   @Override
@@ -87,8 +84,8 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
 
   @Override
   public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception
{
-    if (container == null) {
-      throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "Core container instance
not initialized");
+    if (metricManager == null) {
+      throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "SolrMetricManager instance
not initialized");
     }
 
     handleRequest(req.getParams(), (k, v) -> rsp.add(k, v));
@@ -237,7 +234,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
   public Set<String> parseRegistries(String[] groupStr, String[] registryStr) {
     if ((groupStr == null || groupStr.length == 0) && (registryStr == null || registryStr.length
== 0)) {
       // return all registries
-      return container.getMetricManager().registryNames();
+      return metricManager.registryNames();
     }
     boolean allRegistries = false;
     Set<String> initialPrefixes = Collections.emptySet();
@@ -253,7 +250,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
           initialPrefixes.add(SolrMetricManager.overridableRegistryName(s.trim()));
         }
         if (allRegistries) {
-          return container.getMetricManager().registryNames();
+          return metricManager.registryNames();
         }
       }
     }
@@ -272,12 +269,12 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
           initialPrefixes.add(SolrMetricManager.overridableRegistryName(s.trim()));
         }
         if (allRegistries) {
-          return container.getMetricManager().registryNames();
+          return metricManager.registryNames();
         }
       }
     }
     Set<String> validRegistries = new HashSet<>();
-    for (String r : container.getMetricManager().registryNames()) {
+    for (String r : metricManager.registryNames()) {
       for (String prefix : initialPrefixes) {
         if (r.startsWith(prefix)) {
           validRegistries.add(r);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b5fa6653/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
index 5db09dd..87f3c2b 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
@@ -16,7 +16,9 @@
  */
 package org.apache.solr.handler.admin;
 
+import java.io.Closeable;
 import java.io.IOException;
+import java.lang.invoke.MethodHandles;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -34,7 +36,10 @@ import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.solr.api.Api;
 import org.apache.solr.api.ApiBag;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.cloud.SolrCloudManager;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.common.cloud.ClusterState;
 import org.apache.solr.common.params.CollectionAdminParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.util.NamedList;
@@ -55,11 +60,14 @@ import org.rrd4j.core.RrdBackendFactory;
 import org.rrd4j.core.RrdDb;
 import org.rrd4j.core.RrdDef;
 import org.rrd4j.core.Sample;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  *
  */
-public class MetricsHistoryHandler extends RequestHandlerBase implements PermissionNameProvider
{
+public class MetricsHistoryHandler extends RequestHandlerBase implements PermissionNameProvider,
Closeable {
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
   public static final Set<String> DEFAULT_CORE_COUNTERS = new HashSet<String>()
{{
     add("QUERY./select.requests");
@@ -79,28 +87,32 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements
Permiss
   }};
 
   public static final int DEFAULT_COLLECT_PERIOD = 60;
-  public static final String URI_PREFIX = "solr:///";
+  public static final String URI_PREFIX = "solr://";
 
   private final SolrRrdBackendFactory factory;
+  private final SolrClient solrClient;
   private final MetricsHandler metricsHandler;
   private final SolrMetricManager metricManager;
+  private final SolrCloudManager cloudManager;
   private final ScheduledThreadPoolExecutor collectService;
   private final TimeSource timeSource;
   private final int collectPeriod;
   private final Map<String, Set<String>> counters = new HashMap<>();
   private final Map<String, Set<String>> gauges = new HashMap<>();
 
-  public MetricsHistoryHandler(CoreContainer coreContainer) {
-    factory = new SolrRrdBackendFactory(new CloudSolrClient.Builder(
-        Collections.singletonList(coreContainer.getZkController().getZkServerAddress()),
-        Optional.empty())
-        .withHttpClient(coreContainer.getUpdateShardHandler().getHttpClient())
-        .build(), CollectionAdminParams.SYSTEM_COLL);
+  private boolean logMissingCollection = true;
+
+  public MetricsHistoryHandler(SolrMetricManager metricManager, MetricsHandler metricsHandler,
+                               SolrClient solrClient, SolrCloudManager cloudManager) {
+    factory = new SolrRrdBackendFactory(solrClient, CollectionAdminParams.SYSTEM_COLL,
+        SolrRrdBackendFactory.DEFAULT_SYNC_PERIOD, cloudManager.getTimeSource());
     RrdBackendFactory.registerAndSetAsDefaultFactory(factory);
-    metricsHandler = coreContainer.getMetricsHandler();
-    metricManager = coreContainer.getMetricManager();
+    this.solrClient = solrClient;
+    this.metricsHandler = metricsHandler;
+    this.metricManager = metricManager;
+    this.cloudManager = cloudManager;
     collectPeriod = DEFAULT_COLLECT_PERIOD;
-    timeSource = coreContainer.getZkController().getSolrCloudManager().getTimeSource();
+    this.timeSource = cloudManager.getTimeSource();
 
     counters.put(Group.core.toString(), DEFAULT_CORE_COUNTERS);
     counters.put(Group.node.toString(), Collections.emptySet());
@@ -116,7 +128,26 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements
Permiss
     collectService.scheduleWithFixedDelay(() -> collectMetrics(), collectPeriod, collectPeriod,
TimeUnit.SECONDS);
   }
 
+  public SolrClient getSolrClient() {
+    return solrClient;
+  }
+
   private void collectMetrics() {
+    // check that .system exists
+    try {
+      ClusterState clusterState = cloudManager.getClusterStateProvider().getClusterState();
+      if (clusterState.getCollectionOrNull(CollectionAdminParams.SYSTEM_COLL) == null) {
+        if (logMissingCollection) {
+          log.warn("Missing " + CollectionAdminParams.SYSTEM_COLL + ", skipping metrics collection");
+          logMissingCollection = false;
+        }
+        return;
+      }
+    } catch (Exception e) {
+      log.warn("Error getting cluster state, skipping metrics collection", e);
+      return;
+    }
+    logMissingCollection = true;
     // get metrics
     for (Group group : Arrays.asList(Group.core, Group.node, Group.jvm)) {
       ModifiableSolrParams params = new ModifiableSolrParams();
@@ -148,6 +179,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements
Permiss
             if (db == null) {
               continue;
             }
+            // set the timestamp
             Sample s = db.createSample(TimeUnit.MILLISECONDS.convert(timeSource.getEpochTimeNs(),
TimeUnit.NANOSECONDS));
             NamedList<Object> values = (NamedList<Object>)entry.getValue();
             AtomicBoolean dirty = new AtomicBoolean(false);
@@ -179,16 +211,23 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements
Permiss
   RrdDef createDef(String registry, Group group) {
     registry = SolrMetricManager.overridableRegistryName(registry);
 
+    // base sampling period is collectPeriod - samples more frequent than
+    // that will be dropped, samples less frequent will be interpolated
     RrdDef def = new RrdDef(URI_PREFIX + registry, collectPeriod);
     def.setStartTime(TimeUnit.MILLISECONDS.convert(timeSource.getEpochTimeNs(), TimeUnit.NANOSECONDS));
 
     // add datasources
+
+    // use NaN when more than 1 sample is missing
     counters.get(group.toString()).forEach(c ->
         def.addDatasource(c, DsType.COUNTER, collectPeriod * 2, Double.NaN, Double.NaN));
     gauges.get(group.toString()).forEach(g ->
         def.addDatasource(g, DsType.GAUGE, collectPeriod * 2, Double.NaN, Double.NaN));
 
     // add archives
+
+    // use AVERAGE consolidation,
+    // use NaN when >50% samples are missing
     def.addArchive(ConsolFun.AVERAGE, 0.5, 1, 120); // 2 hours
     def.addArchive(ConsolFun.AVERAGE, 0.5, 10, 288); // 48 hours
     def.addArchive(ConsolFun.AVERAGE, 0.5, 60, 336); // 2 weeks
@@ -197,6 +236,16 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements
Permiss
   }
 
   @Override
+  public void close() {
+    if (collectService != null) {
+      collectService.shutdown();
+    }
+    if (factory != null) {
+      factory.close();
+    }
+  }
+
+  @Override
   public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception
{
 
   }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b5fa6653/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
index c395a53..956aabb 100644
--- a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
+++ b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
@@ -101,6 +101,10 @@ public class SolrRrdBackend extends RrdByteArrayBackend implements Closeable
{
     }
   }
 
+  public void markClean() {
+    dirty = false;
+  }
+
   @Override
   public void close() throws IOException {
     super.close();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b5fa6653/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
index 1401ee3..ca673e4 100644
--- a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
+++ b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
@@ -18,10 +18,17 @@ package org.apache.solr.metrics.rrd;
 
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
+import java.net.URI;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
@@ -42,6 +49,7 @@ import org.apache.solr.common.params.CollectionAdminParams;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.util.IOUtils;
+import org.apache.solr.common.util.TimeSource;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.util.DefaultSolrThreadFactory;
 import org.rrd4j.core.RrdBackend;
@@ -64,21 +72,25 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements
SolrClos
   public static final String DATA_FIELD = "data_bin";
 
   private final SolrClient solrClient;
+  private final TimeSource timeSource;
   private final String collection;
   private ScheduledThreadPoolExecutor syncService;
-  private int syncPeriod = DEFAULT_SYNC_PERIOD;
   private volatile boolean closed = false;
 
   private final Map<String, SolrRrdBackend> backends = new ConcurrentHashMap<>();
 
-  public SolrRrdBackendFactory(SolrClient solrClient, String collection) {
+  public SolrRrdBackendFactory(SolrClient solrClient, String collection, int syncPeriod,
TimeSource timeSource) {
     this.solrClient = solrClient;
+    this.timeSource = timeSource;
     this.collection = collection;
     syncService = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(2,
         new DefaultSolrThreadFactory("SolrRrdBackendFactory"));
     syncService.setRemoveOnCancelPolicy(true);
     syncService.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
-    syncService.scheduleWithFixedDelay(() -> maybeSyncBackends(), syncPeriod, syncPeriod,
TimeUnit.SECONDS);
+    syncService.scheduleWithFixedDelay(() -> maybeSyncBackends(),
+        timeSource.convertDelay(TimeUnit.SECONDS, syncPeriod, TimeUnit.MILLISECONDS),
+        timeSource.convertDelay(TimeUnit.SECONDS, syncPeriod, TimeUnit.MILLISECONDS),
+        TimeUnit.MILLISECONDS);
   }
 
   private void ensureOpen() throws IOException {
@@ -88,6 +100,23 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements
SolrClos
   }
 
   @Override
+  public boolean canStore(URI uri) {
+    if (uri == null) {
+      return false;
+    }
+    if (uri.getScheme().toUpperCase(Locale.ROOT).equals(getName())) {
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  @Override
+  public String getPath(URI uri) {
+    return uri.getSchemeSpecificPart();
+  }
+
+  @Override
   protected synchronized RrdBackend open(String path, boolean readOnly) throws IOException
{
     ensureOpen();
     SolrRrdBackend backend = backends.computeIfAbsent(path, p -> new SolrRrdBackend(p,
readOnly, this));
@@ -143,7 +172,7 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements
SolrClos
   }
 
   public List<String> list() throws IOException {
-    ArrayList<String> names = new ArrayList<>();
+    Set<String> names = new HashSet<>();
     try {
       ModifiableSolrParams params = new ModifiableSolrParams();
       params.add(CommonParams.Q, "*:*");
@@ -159,7 +188,11 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements
SolrClos
     } catch (SolrServerException e) {
       log.warn("Error retrieving RRD list", e);
     }
-    return names;
+    // add in-memory backends not yet stored
+    names.addAll(backends.keySet());
+    ArrayList<String> list = new ArrayList<>(names);
+    Collections.sort(list);
+    return list;
   }
 
   public void remove(String path) throws IOException {
@@ -179,6 +212,7 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements
SolrClos
     if (closed) {
       return;
     }
+    log.info("-- maybe sync backends: " + backends.keySet());
     Map<String, byte[]> syncData = new HashMap<>();
     backends.forEach((path, backend) -> {
       byte[] data = backend.getSyncData();
@@ -189,6 +223,7 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements
SolrClos
     if (syncData.isEmpty()) {
       return;
     }
+    log.info("-- syncing " + syncData.keySet());
     // write updates
     try {
       syncData.forEach((path, data) -> {
@@ -196,6 +231,7 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements
SolrClos
         doc.setField("id", ID_PREFIX + path);
         doc.addField(CommonParams.TYPE, DOC_TYPE);
         doc.addField(DATA_FIELD, data);
+        doc.setField("timestamp", new Date(TimeUnit.MILLISECONDS.convert(timeSource.getEpochTimeNs(),
TimeUnit.NANOSECONDS)));
         try {
           solrClient.add(collection, doc);
         } catch (SolrServerException | IOException e) {
@@ -207,6 +243,12 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements
SolrClos
       } catch (SolrServerException e) {
         log.warn("Error committing RRD data updates", e);
       }
+      syncData.forEach((path, data) -> {
+        SolrRrdBackend backend = backends.get(path);
+        if (backend != null) {
+          backend.markClean();
+        }
+      });
     } catch (IOException e) {
       log.warn("Error sending RRD data updates", e);
     }
@@ -214,6 +256,10 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements
SolrClos
 
   @Override
   protected boolean exists(String path) throws IOException {
+    // check in-memory backends first
+    if (backends.containsKey(path)) {
+      return true;
+    }
     try {
       ModifiableSolrParams params = new ModifiableSolrParams();
       params.add(CommonParams.Q, "{!term f=id}" + ID_PREFIX + path);
@@ -249,7 +295,7 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements
SolrClos
   }
 
   @Override
-  public void close() throws IOException {
+  public void close() {
     closed = true;
     backends.forEach((p, b) -> IOUtils.closeQuietly(b));
     backends.clear();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b5fa6653/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
index 0fe5ad7..392bdfc 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
@@ -48,7 +48,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
 
   @Test
   public void test() throws Exception {
-    MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
+    MetricsHandler handler = new MetricsHandler(h.getCoreContainer().getMetricManager());
 
     SolrQueryResponse resp = new SolrQueryResponse();
     handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", MetricsHandler.COMPACT_PARAM,
"false", CommonParams.WT, "json"), resp);
@@ -179,7 +179,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
 
   @Test
   public void testCompact() throws Exception {
-    MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
+    MetricsHandler handler = new MetricsHandler(h.getCoreContainer().getMetricManager());
 
     SolrQueryResponse resp = new SolrQueryResponse();
     handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
MetricsHandler.COMPACT_PARAM, "true"), resp);
@@ -197,7 +197,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
   public void testPropertyFilter() throws Exception {
     assertQ(req("*:*"), "//result[@numFound='0']");
 
-    MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
+    MetricsHandler handler = new MetricsHandler(h.getCoreContainer().getMetricManager());
 
     SolrQueryResponse resp = new SolrQueryResponse();
     handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
@@ -234,7 +234,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
 
   @Test
   public void testKeyMetrics() throws Exception {
-    MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
+    MetricsHandler handler = new MetricsHandler(h.getCoreContainer().getMetricManager());
 
     String key1 = "solr.core.collection1:CACHE.core.fieldCache";
     SolrQueryResponse resp = new SolrQueryResponse();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b5fa6653/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java
b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java
new file mode 100644
index 0000000..58010fe
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.admin;
+
+import org.apache.solr.SolrTestCaseJ4;
+
+/**
+ *
+ */
+public class MetricsHistoryHandlerTest extends SolrTestCaseJ4 {
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b5fa6653/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
b/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
new file mode 100644
index 0000000..9c74a47
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.metrics.rrd;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CollectionAdminParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.TimeSource;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.rrd4j.ConsolFun;
+import org.rrd4j.DsType;
+import org.rrd4j.core.FetchData;
+import org.rrd4j.core.FetchRequest;
+import org.rrd4j.core.RrdBackendFactory;
+import org.rrd4j.core.RrdDb;
+import org.rrd4j.core.RrdDef;
+import org.rrd4j.core.Sample;
+
+/**
+ *
+ */
+public class SolrRrdBackendFactoryTest extends SolrTestCaseJ4 {
+
+  private SolrRrdBackendFactory factory;
+  private MockSolrClient solrClient;
+  private TimeSource timeSource;
+
+  @Before
+  public void setup() throws Exception {
+    solrClient = new MockSolrClient();
+    if (random().nextBoolean()) {
+      timeSource = TimeSource.NANO_TIME;
+    } else {
+      timeSource = TimeSource.get("simTime:50");
+    }
+    factory = new SolrRrdBackendFactory(solrClient, CollectionAdminParams.SYSTEM_COLL, 1,
timeSource);
+    RrdBackendFactory.registerAndSetAsDefaultFactory(factory);
+  }
+
+  @After
+  public void teardown() throws Exception {
+    if (factory != null) {
+      factory.close();
+    }
+  }
+
+  private RrdDef createDef() {
+    RrdDef def = new RrdDef("solr:foo", 60);
+    def.addDatasource("one", DsType.COUNTER, 120, Double.NaN, Double.NaN);
+    def.addDatasource("two", DsType.GAUGE, 120, Double.NaN, Double.NaN);
+    def.addArchive(ConsolFun.AVERAGE, 0.5, 1, 120); // 2 hours
+    def.addArchive(ConsolFun.AVERAGE, 0.5, 10, 288); // 48 hours
+    def.addArchive(ConsolFun.AVERAGE, 0.5, 60, 336); // 2 weeks
+    def.addArchive(ConsolFun.AVERAGE, 0.5, 240, 180); // 2 months
+    return def;
+  }
+
+  @Test
+  public void testBasic() throws Exception {
+    RrdDb db = new RrdDb(createDef());
+    List<String> list = factory.list();
+    assertEquals(list.toString(), 1, list.size());
+    assertEquals(list.toString(), "foo", list.get(0));
+    timeSource.sleep(2000);
+    // there should be one sync data
+    assertEquals(solrClient.docs.toString(), 1, solrClient.docs.size());
+    SolrInputDocument doc = solrClient.docs.get(SolrRrdBackendFactory.ID_PREFIX + "foo");
+    long timestamp = ((Date)doc.getFieldValue("timestamp")).getTime();
+    timeSource.sleep(2000);
+    SolrInputDocument newDoc = solrClient.docs.get(SolrRrdBackendFactory.ID_PREFIX + "foo");
+    assertEquals(newDoc.toString(), newDoc, doc);
+    long firstTimestamp = TimeUnit.SECONDS.convert(timestamp, TimeUnit.MILLISECONDS);
+    long lastTimestamp = firstTimestamp + 60;
+    // update the db
+    Sample s = db.createSample();
+    for (int i = 0; i < 100; i++) {
+      s.setTime(lastTimestamp);
+      s.setValue("one", 1000 + i * 60);
+      s.setValue("two", 100);
+      s.update();
+      lastTimestamp = lastTimestamp + 60;
+    }
+    timeSource.sleep(3000);
+    newDoc = solrClient.docs.get(SolrRrdBackendFactory.ID_PREFIX + "foo");
+    assertFalse(newDoc.toString(), newDoc.equals(doc));
+    long newTimestamp = ((Date)newDoc.getFieldValue("timestamp")).getTime();
+    assertNotSame(newTimestamp, timestamp);
+    FetchRequest fr = db.createFetchRequest(ConsolFun.AVERAGE, firstTimestamp + 60, lastTimestamp
- 60, 60);
+    FetchData fd = fr.fetchData();
+    int rowCount = fd.getRowCount();
+    double[] one = fd.getValues("one");
+    assertEquals("one", 101, one.length);
+    assertEquals(Double.NaN, one[0], 0.00001);
+    assertEquals(Double.NaN, one[100], 0.00001);
+    for (int i = 1; i < 100; i++) {
+      assertEquals(1.0, one[i], 0.00001);
+    }
+    double[] two = fd.getValues("two");
+    assertEquals(Double.NaN, two[100], 0.00001);
+    for (int i = 0; i < 100; i++) {
+      assertEquals(100.0, two[i], 0.00001);
+    }
+    db.close();
+
+    // should still be listed
+    list = factory.list();
+    assertEquals(list.toString(), 1, list.size());
+    assertEquals(list.toString(), "foo", list.get(0));
+
+    // re-open read-write
+    db = new RrdDb("solr:foo");
+    s = db.createSample();
+    s.setTime(lastTimestamp);
+    s.setValue("one", 7000);
+    s.setValue("two", 100);
+    s.update();
+    timeSource.sleep(3000);
+    // should update
+    timestamp = newTimestamp;
+    doc = newDoc;
+    newDoc = solrClient.docs.get(SolrRrdBackendFactory.ID_PREFIX + "foo");
+    assertFalse(newDoc.toString(), newDoc.equals(doc));
+    newTimestamp = ((Date)newDoc.getFieldValue("timestamp")).getTime();
+    assertNotSame(newTimestamp, timestamp);
+    fr = db.createFetchRequest(ConsolFun.AVERAGE, firstTimestamp + 60, lastTimestamp, 60);
+    fd = fr.fetchData();
+    rowCount = fd.getRowCount();
+    one = fd.getValues("one");
+    assertEquals("one", 102, one.length);
+    assertEquals(Double.NaN, one[0], 0.00001);
+    assertEquals(Double.NaN, one[101], 0.00001);
+    for (int i = 1; i < 101; i++) {
+      assertEquals(1.0, one[i], 0.00001);
+    }
+    two = fd.getValues("two");
+    assertEquals(Double.NaN, two[101], 0.00001);
+    for (int i = 0; i < 101; i++) {
+      assertEquals(100.0, two[i], 0.00001);
+    }
+
+    // open a read-only version of the db
+    RrdDb readOnly = new RrdDb("solr:foo", true);
+    s = readOnly.createSample();
+    s.setTime(lastTimestamp + 120);
+    s.setValue("one", 10000001);
+    s.setValue("two", 100);
+    s.update();
+    // these updates should not be synced
+    timeSource.sleep(3000);
+    doc = newDoc;
+    timestamp = newTimestamp;
+    newDoc = solrClient.docs.get(SolrRrdBackendFactory.ID_PREFIX + "foo");
+    assertTrue(newDoc.toString(), newDoc.equals(doc));
+    newTimestamp = ((Date)newDoc.getFieldValue("timestamp")).getTime();
+    assertEquals(newTimestamp, timestamp);
+  }
+
+  static class MockSolrClient extends SolrClient {
+    LinkedHashMap<String, SolrInputDocument> docs = new LinkedHashMap<>();
+
+    @Override
+    public NamedList<Object> request(SolrRequest request, String collection) throws
SolrServerException, IOException {
+      NamedList<Object> res = new NamedList<>();
+      if (request instanceof UpdateRequest) {
+        List<SolrInputDocument> docList = ((UpdateRequest)request).getDocuments();
+        if (docList != null) {
+          assertEquals(docList.toString(), 1, docList.size());
+          SolrInputDocument doc = docList.get(0);
+          String id = (String)doc.getFieldValue("id");
+          assertNotNull(doc.toString(), id);
+          docs.put(id, doc);
+        }
+      } else if (request instanceof QueryRequest) {
+        SolrParams params = request.getParams();
+        String query = params.get("q");
+        final SolrDocumentList lst = new SolrDocumentList();
+        if (query != null) {
+          if (query.startsWith("{!term f=id}")) {
+            String id = query.substring(12);
+            SolrInputDocument doc = docs.get(id);
+            if (doc != null) {
+              SolrDocument d = new SolrDocument();
+              doc.forEach((k, f) -> {
+                f.forEach(v -> d.addField(k, v));
+              });
+              lst.add(d);
+              lst.setNumFound(1);
+            }
+          } else if (query.equals("*:*")) {
+            if (!docs.isEmpty()) {
+              lst.setNumFound(docs.size());
+              docs.values().forEach(doc -> {
+                SolrDocument d = new SolrDocument();
+                doc.forEach((k, f) -> {
+                  f.forEach(v -> d.addField(k, v));
+                });
+                lst.add(d);
+              });
+            }
+          }
+        }
+        res.add("response", lst);
+      }
+      return res;
+    }
+
+    @Override
+    public void close() throws IOException {
+
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b5fa6653/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
index a67f433..899214d 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
@@ -183,6 +183,7 @@ public interface CommonParams {
   String AUTHC_PATH = "/admin/authentication";
   String ZK_PATH = "/admin/zookeeper";
   String METRICS_PATH = "/admin/metrics";
+  String METRICS_HISTORY_PATH = "/admin/metrics/history";
   String AUTOSCALING_PATH = "/admin/autoscaling";
   String AUTOSCALING_HISTORY_PATH = "/admin/autoscaling/history";
   String AUTOSCALING_DIAGNOSTICS_PATH = "/admin/autoscaling/diagnostics";


Mime
View raw message