lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject lucene-solr:jira/solr-12438: SOLR-12438: Several improvements: * report "mode" * works also in non-cloud setup * automatically pull history from Overseer when in memory mode.
Date Tue, 05 Jun 2018 19:48:23 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/jira/solr-12438 [created] 2fa260c14


SOLR-12438: Several improvements:
* report "mode"
* works also in non-cloud setup
* automatically pull history from Overseer when in memory mode.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2fa260c1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2fa260c1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2fa260c1

Branch: refs/heads/jira/solr-12438
Commit: 2fa260c14cb57412f4d250c4d642dfa256287d6e
Parents: 6624773
Author: Andrzej Bialecki <ab@apache.org>
Authored: Tue Jun 5 21:46:12 2018 +0200
Committer: Andrzej Bialecki <ab@apache.org>
Committed: Tue Jun 5 21:46:12 2018 +0200

----------------------------------------------------------------------
 .../org/apache/solr/core/CoreContainer.java     |  58 +++--
 .../handler/admin/MetricsHistoryHandler.java    | 219 +++++++++++++------
 .../apache/solr/metrics/rrd/SolrRrdBackend.java |  30 ++-
 .../solr/metrics/rrd/SolrRrdBackendFactory.java |  76 +++++--
 .../cloud/MetricsHistoryIntegrationTest.java    |   9 +-
 .../cloud/autoscaling/sim/SimCloudManager.java  |   1 +
 .../admin/MetricsHistoryHandlerTest.java        |   9 +-
 .../metrics/rrd/SolrRrdBackendFactoryTest.java  |  16 +-
 8 files changed, 296 insertions(+), 122 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2fa260c1/solr/core/src/java/org/apache/solr/core/CoreContainer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index d546dd2..5cbcab5 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -26,6 +26,7 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -43,6 +44,9 @@ import org.apache.http.client.CredentialsProvider;
 import org.apache.http.config.Lookup;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.store.Directory;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
 import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
@@ -59,6 +63,7 @@ import org.apache.solr.common.SolrException.ErrorCode;
 import org.apache.solr.common.cloud.DocCollection;
 import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.Replica.State;
+import org.apache.solr.common.params.CollectionAdminParams;
 import org.apache.solr.common.util.ExecutorUtil;
 import org.apache.solr.common.util.IOUtils;
 import org.apache.solr.common.util.Utils;
@@ -570,21 +575,7 @@ public class CoreContainer {
     containerHandlers.put(METRICS_PATH, metricsHandler);
     metricsHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag,
METRICS_PATH);
 
-    if (isZooKeeperAware()) {
-      PluginInfo plugin = cfg.getMetricsConfig().getHistoryHandler();
-      Map<String, Object> initArgs;
-      if (plugin != null && plugin.initArgs != null) {
-        initArgs = plugin.initArgs.asMap(5);
-        initArgs.put(MetricsHistoryHandler.ENABLE_PROP, plugin.isEnabled());
-      } else {
-        initArgs = Collections.emptyMap();
-      }
-      metricsHistoryHandler = new MetricsHistoryHandler(getZkController().getNodeName(),
metricsHandler,
-          new CloudSolrClient.Builder(Collections.singletonList(getZkController().getZkServerAddress()),
Optional.empty())
-      .withHttpClient(updateShardHandler.getDefaultHttpClient()).build(), getZkController().getSolrCloudManager(),
initArgs);
-      containerHandlers.put(METRICS_HISTORY_PATH, metricsHistoryHandler);
-      metricsHistoryHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(),
metricTag, METRICS_HISTORY_PATH);
-    }
+    createMetricsHistoryHandler();
 
     autoscalingHistoryHandler = createHandler(AUTOSCALING_HISTORY_PATH, AutoscalingHistoryHandler.class.getName(),
AutoscalingHistoryHandler.class);
     metricsCollectorHandler = createHandler(MetricsCollectorHandler.HANDLER_PATH, MetricsCollectorHandler.class.getName(),
MetricsCollectorHandler.class);
@@ -748,6 +739,43 @@ public class CoreContainer {
     status |= LOAD_COMPLETE | INITIAL_CORE_LOAD_COMPLETE;
   }
 
+  private void createMetricsHistoryHandler() {
+    PluginInfo plugin = cfg.getMetricsConfig().getHistoryHandler();
+    Map<String, Object> initArgs;
+    if (plugin != null && plugin.initArgs != null) {
+      initArgs = plugin.initArgs.asMap(5);
+      initArgs.put(MetricsHistoryHandler.ENABLE_PROP, plugin.isEnabled());
+    } else {
+      initArgs = new HashMap<>();
+    }
+    String name;
+    SolrCloudManager cloudManager;
+    SolrClient client;
+    if (isZooKeeperAware()) {
+      name = getZkController().getNodeName();
+      cloudManager = getZkController().getSolrCloudManager();
+      client = new CloudSolrClient.Builder(Collections.singletonList(getZkController().getZkServerAddress()),
Optional.empty())
+          .withHttpClient(updateShardHandler.getDefaultHttpClient()).build();
+    } else {
+      name = getNodeConfig().getNodeName();
+      if (name == null || name.isEmpty()) {
+        name = "localhost";
+      }
+      cloudManager = null;
+      client = new EmbeddedSolrServer(this, CollectionAdminParams.SYSTEM_COLL);
+      // enable local metrics
+      if (!initArgs.containsKey(MetricsHistoryHandler.ENABLE_NODES_PROP)) {
+        initArgs.put(MetricsHistoryHandler.ENABLE_NODES_PROP, true);
+      }
+      if (!initArgs.containsKey(MetricsHistoryHandler.ENABLE_REPLICAS_PROP)) {
+        initArgs.put(MetricsHistoryHandler.ENABLE_REPLICAS_PROP, true);
+      }
+    }
+    metricsHistoryHandler = new MetricsHistoryHandler(name, metricsHandler,
+        client, cloudManager, initArgs);
+    containerHandlers.put(METRICS_HISTORY_PATH, metricsHistoryHandler);
+    metricsHistoryHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(),
metricTag, METRICS_HISTORY_PATH);
+  }
 
   public void securityNodeChanged() {
     log.info("Security node changed, reloading security.json");

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2fa260c1/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
index 3d48680..89731d2 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
@@ -19,6 +19,7 @@ package org.apache.solr.handler.admin;
 import javax.imageio.ImageIO;
 import java.awt.Color;
 import java.awt.image.BufferedImage;
+import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.Closeable;
 import java.io.IOException;
@@ -51,11 +52,14 @@ import com.google.common.annotations.VisibleForTesting;
 import org.apache.solr.api.Api;
 import org.apache.solr.api.ApiBag;
 import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.cloud.NodeStateProvider;
 import org.apache.solr.client.solrj.cloud.SolrCloudManager;
 import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
 import org.apache.solr.client.solrj.cloud.autoscaling.Suggestion;
 import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
 import org.apache.solr.cloud.Overseer;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.cloud.ClusterState;
@@ -68,9 +72,12 @@ import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.Base64;
+import org.apache.solr.common.util.JavaBinCodec;
 import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.Pair;
 import org.apache.solr.common.util.SimpleOrderedMap;
 import org.apache.solr.common.util.TimeSource;
+import org.apache.solr.common.util.Utils;
 import org.apache.solr.handler.RequestHandlerBase;
 import org.apache.solr.metrics.SolrMetricManager;
 import org.apache.solr.metrics.rrd.SolrRrdBackendFactory;
@@ -148,6 +155,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements
Permiss
   private final int collectPeriod;
   private final Map<String, List<String>> counters = new HashMap<>();
   private final Map<String, List<String>> gauges = new HashMap<>();
+  private final String overseerUrlScheme;
 
   private final Map<String, RrdDb> knownDbs = new ConcurrentHashMap<>();
 
@@ -166,11 +174,17 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements
Permiss
     if (pluginArgs != null) {
       args.putAll(pluginArgs);
     }
-    // override from ZK
-    Map<String, Object> props = (Map<String, Object>)cloudManager.getClusterStateProvider()
-        .getClusterProperty("metrics", Collections.emptyMap())
-        .getOrDefault("history", Collections.emptyMap());
-    args.putAll(props);
+    // override from ZK if available
+    if (cloudManager != null) {
+      Map<String, Object> props = (Map<String, Object>)cloudManager.getClusterStateProvider()
+          .getClusterProperty("metrics", Collections.emptyMap())
+          .getOrDefault("history", Collections.emptyMap());
+      args.putAll(props);
+
+      overseerUrlScheme = cloudManager.getClusterStateProvider().getClusterProperty("urlScheme",
"http");
+    } else {
+      overseerUrlScheme = "http";
+    }
 
     this.nodeName = nodeName;
     this.enable = Boolean.parseBoolean(String.valueOf(args.getOrDefault(ENABLE_PROP, "true")));
@@ -180,12 +194,12 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements
Permiss
     this.collectPeriod = Integer.parseInt(String.valueOf(args.getOrDefault(COLLECT_PERIOD_PROP,
DEFAULT_COLLECT_PERIOD)));
     int syncPeriod = Integer.parseInt(String.valueOf(args.getOrDefault(SYNC_PERIOD_PROP,
SolrRrdBackendFactory.DEFAULT_SYNC_PERIOD)));
 
-    factory = new SolrRrdBackendFactory(solrClient, CollectionAdminParams.SYSTEM_COLL,
-            syncPeriod, cloudManager.getTimeSource());
     this.solrClient = solrClient;
     this.metricsHandler = metricsHandler;
     this.cloudManager = cloudManager;
-    this.timeSource = cloudManager.getTimeSource();
+    this.timeSource = cloudManager != null ? cloudManager.getTimeSource() : TimeSource.NANO_TIME;
+    factory = new SolrRrdBackendFactory(solrClient, CollectionAdminParams.SYSTEM_COLL,
+            syncPeriod, this.timeSource);
 
     counters.put(Group.core.toString(), DEFAULT_CORE_COUNTERS);
     counters.put(Group.node.toString(), Collections.emptyList());
@@ -217,43 +231,60 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements
Permiss
     }
   }
 
+  // check that .system exists
   public void checkSystemCollection() {
-    // check that .system exists
-    try {
-      if (cloudManager.isClosed() || Thread.interrupted()) {
+    if (cloudManager != null) {
+      try {
+        if (cloudManager.isClosed() || Thread.interrupted()) {
+          factory.setPersistent(false);
+          return;
+        }
+        ClusterState clusterState = cloudManager.getClusterStateProvider().getClusterState();
+        DocCollection systemColl = clusterState.getCollectionOrNull(CollectionAdminParams.SYSTEM_COLL);
+        if (systemColl == null) {
+          if (logMissingCollection) {
+            log.warn("Missing " + CollectionAdminParams.SYSTEM_COLL + ", keeping metrics
history in memory");
+            logMissingCollection = false;
+          }
+          factory.setPersistent(false);
+          return;
+        } else {
+          boolean ready = false;
+          for (Replica r : systemColl.getReplicas()) {
+            if (r.isActive(clusterState.getLiveNodes())) {
+              ready = true;
+              break;
+            }
+          }
+          if (!ready) {
+            log.debug(CollectionAdminParams.SYSTEM_COLL + " not ready yet, keeping metrics
history in memory");
+            factory.setPersistent(false);
+            return;
+          }
+        }
+      } catch (Exception e) {
+        if (logMissingCollection) {
+          log.warn("Error getting cluster state, keeping metrics history in memory", e);
+        }
+        logMissingCollection = false;
         factory.setPersistent(false);
         return;
       }
-      ClusterState clusterState = cloudManager.getClusterStateProvider().getClusterState();
-      DocCollection systemColl = clusterState.getCollectionOrNull(CollectionAdminParams.SYSTEM_COLL);
-      if (systemColl == null) {
+      logMissingCollection = true;
+      factory.setPersistent(true);
+    } else {
+      try {
+        solrClient.query(CollectionAdminParams.SYSTEM_COLL, new SolrQuery(CommonParams.Q,
"*:*", CommonParams.ROWS, "0"));
+        factory.setPersistent(true);
+        logMissingCollection = true;
+      } catch (Exception e) {
         if (logMissingCollection) {
-          log.warn("Missing " + CollectionAdminParams.SYSTEM_COLL + ", keeping metrics history
in memory");
-          logMissingCollection = false;
+          log.warn("Error querying .system collection, keeping metrics history in memory",
e);
         }
+        logMissingCollection = false;
         factory.setPersistent(false);
-        return;
-      } else {
-        boolean ready = false;
-        for (Replica r : systemColl.getReplicas()) {
-          if (r.isActive(clusterState.getLiveNodes())) {
-            ready = true;
-            break;
-          }
-        }
-        if (!ready) {
-          log.debug(CollectionAdminParams.SYSTEM_COLL + " not ready yet, keeping metrics
history in memory");
-          factory.setPersistent(false);
-          return;
-        }
       }
-    } catch (Exception e) {
-      log.warn("Error getting cluster state, keeping metrics history in memory", e);
-      factory.setPersistent(false);
-      return;
     }
-    logMissingCollection = true;
-    factory.setPersistent(true);
   }
 
   public SolrClient getSolrClient() {
@@ -271,7 +302,11 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements
Permiss
     return factory;
   }
 
-  private boolean isOverseerLeader() {
+  private String getOverseerLeader() {
+    // non-ZK node has no Overseer
+    if (cloudManager == null) {
+      return null;
+    }
     ZkNodeProps props = null;
     try {
       VersionedData data = cloudManager.getDistribStateManager().getData(
@@ -281,24 +316,39 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements
Permiss
       }
     } catch (KeeperException | IOException | NoSuchElementException e) {
       log.warn("Could not obtain overseer's address, skipping.", e);
-      return false;
+      return null;
     } catch (InterruptedException e) {
       Thread.currentThread().interrupt();
-      return false;
+      return null;
     }
     if (props == null) {
-      return false;
+      return null;
     }
     String oid = props.getStr(ID);
     if (oid == null) {
-      return false;
+      return null;
     }
     String[] ids = oid.split("-");
     if (ids.length != 3) { // unknown format
       log.warn("Unknown format of leader id, skipping: " + oid);
+      return null;
+    }
+    return ids[1];
+  }
+
+  private boolean amIOverseerLeader() {
+    return amIOverseerLeader(null);
+  }
+
+  private boolean amIOverseerLeader(String leader) {
+    if (leader == null) {
+      leader = getOverseerLeader();
+    }
+    if (leader == null) {
       return false;
+    } else {
+      return nodeName.equals(leader);
     }
-    return nodeName.equals(ids[1]);
   }
 
   private void collectMetrics() {
@@ -383,7 +433,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements
Permiss
   }
 
   private void collectGlobalMetrics() {
-    if (!isOverseerLeader()) {
+    if (!amIOverseerLeader()) {
       return;
     }
     Set<String> nodes = new HashSet<>(cloudManager.getClusterStateProvider().getLiveNodes());
@@ -640,11 +690,18 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements
Permiss
     if (cmd == null) {
       throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "unknown 'action' param
'" + actionStr + "', supported actions: " + Cmd.actions);
     }
-    Object res = null;
+    final SimpleOrderedMap<Object> res = new SimpleOrderedMap<>();
+    rsp.add("metrics", res);
     switch (cmd) {
       case LIST:
         int rows = req.getParams().getInt(CommonParams.ROWS, SolrRrdBackendFactory.DEFAULT_MAX_DBS);
-        res = factory.list(rows);
+        List<Pair<String, Long>> lst = factory.list(rows);
+        lst.forEach(p -> {
+          SimpleOrderedMap<Object> data = new SimpleOrderedMap<>();
+          data.add("lastModified", p.second());
+          data.add("node", nodeName);
+          res.add(p.first(), data);
+        });
         break;
       case GET:
         String name = req.getParams().get(CommonParams.NAME);
@@ -657,15 +714,14 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements
Permiss
         if (format == null) {
           throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "unknown 'format'
param '" + formatStr + "', supported formats: " + Format.formats);
         }
-        if (!factory.exists(name)) {
-          rsp.add("error", "'" + name + "' doesn't exist");
-        } else {
+        if (factory.exists(name)) {
           // get a throwaway copy (safe to close and discard)
           RrdDb db = new RrdDb(URI_PREFIX + name, true, factory);
-          res = new NamedList<>();
-          NamedList<Object> data = new NamedList<>();
+          SimpleOrderedMap<Object> data = new SimpleOrderedMap<>();
           data.add("data", getDbData(db, dsNames, format, req.getParams()));
-          ((NamedList)res).add(name, data);
+          data.add("lastModified", db.getLastUpdateTime());
+          data.add("node", nodeName);
+          res.add(name, data);
           db.close();
         }
         break;
@@ -674,17 +730,14 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements
Permiss
         if (name == null) {
           throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'name' is a required
param");
         }
-        if (!factory.exists(name)) {
-          rsp.add("error", "'" + name + "' doesn't exist");
-        } else {
+        if (factory.exists(name)) {
           // get a throwaway copy (safe to close and discard)
           RrdDb db = new RrdDb(URI_PREFIX + name, true, factory);
-          NamedList<Object> map = new NamedList<>();
-          NamedList<Object> status = new NamedList<>();
+          SimpleOrderedMap<Object> status = new SimpleOrderedMap<>();
           status.add("status", getDbStatus(db));
-          map.add(name, status);
+          status.add("node", nodeName);
+          res.add(name, status);
           db.close();
-          res = map;
         }
         break;
       case DELETE:
@@ -700,9 +753,51 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements
Permiss
         rsp.add("success", "ok");
         break;
     }
-    if (res != null) {
-      rsp.add("metrics", res);
+    // when using in-memory DBs non-overseer node has no access to overseer DBs
+    // forward the request to Overseer leader if available
+    if (!factory.isPersistent()) {
+      String leader = getOverseerLeader();
+      if (leader != null && !amIOverseerLeader(leader)) {
+        // get & merge remote response
+        NamedList<Object> remoteRes = handleRemoteRequest(leader, req);
+        mergeRemoteRes(rsp, remoteRes);
+      }
+    }
+    SimpleOrderedMap<Object> apiState = new SimpleOrderedMap<>();
+    apiState.add("enableReplicas", enableReplicas);
+    apiState.add("enableNodes", enableNodes);
+    apiState.add("mode", enable ? (factory.isPersistent() ? "index" : "memory") : "disabled");
+    if (!factory.isPersistent()) {
+      apiState.add("message", "WARNING: metrics history is not being persisted. Create .system
collection to start persisting history.");
+    }
+    rsp.add("state", apiState);
+    rsp.getResponseHeader().add("zkConnected", cloudManager != null);
+  }
+
+  private NamedList<Object> handleRemoteRequest(String nodeName, SolrQueryRequest req)
{
+    String url = Utils.getBaseUrlForNodeName(nodeName, overseerUrlScheme);
+    ModifiableSolrParams params = new ModifiableSolrParams(req.getParams());
+    params.set(CommonParams.WT, "javabin");
+    url = url + req.getPath() + "?" + params.toString();
+    try {
+      byte[] data = cloudManager.httpRequest(url, SolrRequest.METHOD.GET, null, null, HttpClientUtil.DEFAULT_CONNECT_TIMEOUT,
true);
+      // response is always a NamedList
+      try (JavaBinCodec codec = new JavaBinCodec()) {
+        return (NamedList<Object>)codec.unmarshal(new ByteArrayInputStream(data));
+      }
+    } catch (IOException e) {
+      log.warn("Exception forwarding request to Overseer at " + url, e);
+      return null;
+    }
+  }
+
+  private void mergeRemoteRes(SolrQueryResponse rsp, NamedList<Object> remoteRes) {
+    if (remoteRes == null || remoteRes.get("metrics") == null) {
+      return;
     }
+    NamedList<Object> remoteMetrics = (NamedList<Object>)remoteRes.get("metrics");
+    SimpleOrderedMap localMetrics = (SimpleOrderedMap) rsp.getValues().get("metrics");
+    remoteMetrics.forEach((k, v) -> localMetrics.add(k, v));
   }
 
   private NamedList<Object> getDbStatus(RrdDb db) throws IOException {
@@ -750,7 +845,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements
Permiss
     RrdDef def = db.getRrdDef();
     ArcDef[] arcDefs = def.getArcDefs();
     for (ArcDef arcDef : arcDefs) {
-      SimpleOrderedMap map = new SimpleOrderedMap();
+      SimpleOrderedMap<Object> map = new SimpleOrderedMap<>();
       res.add(arcDef.dump(), map);
       Archive a = db.getArchive(arcDef.getConsolFun(), arcDef.getSteps());
       // startTime / endTime, arcStep are in seconds
@@ -776,7 +871,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements
Permiss
           map.add("timestamps", str.toString());
         }
       }
-      SimpleOrderedMap values = new SimpleOrderedMap();
+      SimpleOrderedMap<Object> values = new SimpleOrderedMap<>();
       map.add("values", values);
       for (String name : dsNames) {
         double[] vals = fd.getValues(name);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2fa260c1/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
index 956aabb..d0aa3e2 100644
--- a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
+++ b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
@@ -19,6 +19,7 @@ package org.apache.solr.metrics.rrd;
 import java.io.Closeable;
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReentrantLock;
 
 import org.rrd4j.core.RrdByteArrayBackend;
@@ -36,14 +37,27 @@ public class SolrRrdBackend extends RrdByteArrayBackend implements Closeable
{
   private final ReentrantLock lock = new ReentrantLock();
   private volatile boolean dirty = false;
   private volatile boolean closed = false;
+  private volatile long lastModifiedTime;
+
+  public static final class SyncData {
+    public byte[] data;
+    public long timestamp;
+
+    public SyncData(byte[] data, long timestamp) {
+      this.data = data;
+      this.timestamp = timestamp;
+    }
+  }
 
   public SolrRrdBackend(String path, boolean readOnly, SolrRrdBackendFactory factory) {
     super(path);
     this.factory = factory;
+    this.lastModifiedTime = TimeUnit.MILLISECONDS.convert(factory.getTimeSource().getEpochTimeNs(),
TimeUnit.NANOSECONDS);
     try {
-      byte[] data = factory.getData(path);
-      if (data != null) {
-        this.buffer = data;
+      SyncData syncData = factory.getData(path);
+      if (syncData != null) {
+        this.buffer = syncData.data;
+        this.lastModifiedTime = syncData.timestamp;
       }
     } catch (IOException e) {
       log.warn("Exception retrieving data from " + path + ", store will be readOnly", e);
@@ -60,6 +74,7 @@ public class SolrRrdBackend extends RrdByteArrayBackend implements Closeable
{
     super(other.getPath());
     readOnly = true;
     factory = null;
+    this.lastModifiedTime = other.lastModifiedTime;
     byte[] otherBuffer = other.buffer;
     buffer = new byte[otherBuffer.length];
     System.arraycopy(otherBuffer, 0, buffer, 0, otherBuffer.length);
@@ -69,6 +84,10 @@ public class SolrRrdBackend extends RrdByteArrayBackend implements Closeable
{
     return readOnly;
   }
 
+  public long getLastModifiedTime() {
+    return lastModifiedTime;
+  }
+
   @Override
   protected void write(long offset, byte[] bytes) throws IOException {
     if (readOnly || closed) {
@@ -77,13 +96,14 @@ public class SolrRrdBackend extends RrdByteArrayBackend implements Closeable
{
     lock.lock();
     try {
       super.write(offset, bytes);
+      lastModifiedTime = TimeUnit.MILLISECONDS.convert(factory.getTimeSource().getEpochTimeNs(),
TimeUnit.NANOSECONDS);
       dirty = true;
     } finally {
       lock.unlock();
     }
   }
 
-  public byte[] getSyncData() {
+  public SyncData getSyncData() {
     if (readOnly || closed) {
       return null;
     }
@@ -95,7 +115,7 @@ public class SolrRrdBackend extends RrdByteArrayBackend implements Closeable
{
     try {
       byte[] bufferCopy = new byte[buffer.length];
       System.arraycopy(buffer, 0, bufferCopy, 0, buffer.length);
-      return bufferCopy;
+      return new SyncData(bufferCopy, lastModifiedTime);
     } finally {
       lock.unlock();
     }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2fa260c1/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
index 06ab5fe..a3c6f64 100644
--- a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
+++ b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
@@ -22,14 +22,12 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Date;
+import java.util.Comparator;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
@@ -47,6 +45,7 @@ import org.apache.solr.common.params.CollectionAdminParams;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.util.IOUtils;
+import org.apache.solr.common.util.Pair;
 import org.apache.solr.common.util.TimeSource;
 import org.apache.solr.util.DefaultSolrThreadFactory;
 import org.rrd4j.core.RrdBackend;
@@ -114,6 +113,10 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements
SolrClos
         TimeUnit.MILLISECONDS);
   }
 
+  public TimeSource getTimeSource() {
+    return timeSource;
+  }
+
   private void ensureOpen() throws IOException {
     if (closed) {
       throw new IOException("Factory already closed");
@@ -181,7 +184,7 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements
SolrClos
     }
   }
 
-  byte[] getData(String path) throws IOException {
+  SolrRrdBackend.SyncData getData(String path) throws IOException {
     if (!persistent) {
       return null;
     }
@@ -203,7 +206,8 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements
SolrClos
         return null;
       }
       if (o instanceof byte[]) {
-        return (byte[])o;
+        Long time = (Long)doc.getFieldValue("timestamp_l");
+        return new SolrRrdBackend.SyncData((byte[])o, time);
       } else {
         throw new SolrServerException("Unexpected value of '" + DATA_FIELD + "' field: "
+ o.getClass().getName() + ": " + o);
       }
@@ -216,34 +220,58 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements
SolrClos
     backends.remove(path);
   }
 
+  private static final class DbComparator implements Comparator<Pair<String, Long>>
{
+    static final DbComparator INSTANCE = new DbComparator();
+
+    @Override
+    public int compare(Pair<String, Long> o1, Pair<String, Long> o2) {
+      return o1.first().compareTo(o2.first());
+    }
+  }
+
   /**
    * List all available databases created by this node name
    * @param maxLength maximum number of results to return
-   * @return list of database names, or empty
+   * @return list of database names and their last update times, or empty
    * @throws IOException on server errors
    */
-  public List<String> list(int maxLength) throws IOException {
-    Set<String> names = new HashSet<>();
+  public List<Pair<String, Long>> list(int maxLength) throws IOException {
+    Map<String, Pair<String, Long>> byName = new HashMap<>();
     if (persistent) {
       try {
         ModifiableSolrParams params = new ModifiableSolrParams();
         params.add(CommonParams.Q, "*:*");
         params.add(CommonParams.FQ, CommonParams.TYPE + ":" + DOC_TYPE);
-        params.add(CommonParams.FL, "id");
+        params.add(CommonParams.FL, "id,timestamp_l");
         params.add(CommonParams.ROWS, String.valueOf(maxLength));
         QueryResponse rsp = solrClient.query(collection, params);
         SolrDocumentList docs = rsp.getResults();
         if (docs != null) {
-          docs.forEach(d -> names.add(((String)d.getFieldValue("id")).substring(idPrefixLength)));
+          docs.forEach(d -> {
+            Long time = (Long)d.getFieldValue("timestamp_l");
+            Pair<String, Long> p = new Pair<>(((String)d.getFieldValue("id")).substring(idPrefixLength),
time);
+            byName.put(p.first(), p);
+          });
         }
       } catch (SolrServerException e) {
         log.warn("Error retrieving RRD list", e);
       }
     }
-    // add in-memory backends not yet stored
-    names.addAll(backends.keySet());
-    ArrayList<String> list = new ArrayList<>(names);
-    Collections.sort(list);
+    // add in-memory backends not yet stored, or replace with more recent versions
+    backends.forEach((name, db) -> {
+      long lastModifiedTime = db.getLastModifiedTime();
+      Pair<String, Long> stored = byName.get(name);
+      Pair<String, Long> inMemory = new Pair(name, lastModifiedTime);
+      if (stored != null) {
+        if (stored.second() < lastModifiedTime) {
+          byName.put(name, inMemory);
+        }
+      } else {
+        byName.put(name, inMemory);
+      }
+    });
+    ArrayList<Pair<String, Long>> list = new ArrayList<>(byName.values());
+    Collections.sort(list, DbComparator.INSTANCE);
     return list;
   }
 
@@ -301,25 +329,25 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements
SolrClos
       return;
     }
     log.debug("-- maybe sync backends: " + backends.keySet());
-    Map<String, byte[]> syncData = new HashMap<>();
+    Map<String, SolrRrdBackend.SyncData> syncDatas = new HashMap<>();
     backends.forEach((path, backend) -> {
-      byte[] data = backend.getSyncData();
-      if (data != null) {
-        syncData.put(backend.getPath(), data);
+      SolrRrdBackend.SyncData syncData = backend.getSyncData();
+      if (syncData != null) {
+        syncDatas.put(backend.getPath(), syncData);
       }
     });
-    if (syncData.isEmpty()) {
+    if (syncDatas.isEmpty()) {
       return;
     }
-    log.debug("-- syncing " + syncData.keySet());
+    log.debug("-- syncing " + syncDatas.keySet());
     // write updates
     try {
-      syncData.forEach((path, data) -> {
+      syncDatas.forEach((path, syncData) -> {
         SolrInputDocument doc = new SolrInputDocument();
         doc.setField("id", ID_PREFIX + ID_SEP + path);
         doc.addField(CommonParams.TYPE, DOC_TYPE);
-        doc.addField(DATA_FIELD, data);
-        doc.setField("timestamp", new Date(TimeUnit.MILLISECONDS.convert(timeSource.getEpochTimeNs(),
TimeUnit.NANOSECONDS)));
+        doc.addField(DATA_FIELD, syncData.data);
+        doc.setField("timestamp_l", syncData.timestamp);
         try {
           solrClient.add(collection, doc);
         } catch (SolrServerException | IOException e) {
@@ -334,7 +362,7 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements
SolrClos
       } catch (SolrServerException e) {
         log.warn("Error committing RRD data updates", e);
       }
-      syncData.forEach((path, data) -> {
+      syncDatas.forEach((path, data) -> {
         SolrRrdBackend backend = backends.get(path);
         if (backend != null) {
           backend.markClean();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2fa260c1/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryIntegrationTest.java
index 2012a1a..b3a1fb6 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryIntegrationTest.java
@@ -35,6 +35,7 @@ import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.Base64;
 import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
 import org.apache.solr.common.util.TimeSource;
 import org.apache.solr.util.LogLevel;
 import org.junit.AfterClass;
@@ -94,12 +95,12 @@ public class MetricsHistoryIntegrationTest extends SolrCloudTestCase {
     NamedList<Object> rsp = solrClient.request(createHistoryRequest(params(CommonParams.ACTION,
"list")));
     assertNotNull(rsp);
     // expected solr.jvm, solr.node and solr.collection..system
-    List<String> lst = (List<String>)rsp.get("metrics");
+    SimpleOrderedMap<Object> lst = (SimpleOrderedMap<Object>) rsp.get("metrics");
     assertNotNull(lst);
     assertEquals(lst.toString(), 3, lst.size());
-    assertTrue(lst.toString(), lst.contains("solr.jvm"));
-    assertTrue(lst.toString(), lst.contains("solr.node"));
-    assertTrue(lst.toString(), lst.contains("solr.collection..system"));
+    assertNotNull(lst.toString(), lst.get("solr.jvm"));
+    assertNotNull(lst.toString(), lst.get("solr.node"));
+    assertNotNull(lst.toString(), lst.get("solr.collection..system"));
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2fa260c1/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
index 234eaea..900fc76 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
@@ -665,6 +665,7 @@ public class SimCloudManager implements SolrCloudManager {
         }
         queryRequest.getContext().put("httpMethod", req.getMethod().toString());
         SolrQueryResponse queryResponse = new SolrQueryResponse();
+        queryResponse.addResponseHeader(new SimpleOrderedMap<>());
         if (autoscaling) {
           autoScalingHandler.handleRequest(queryRequest, queryResponse);
         } else {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2fa260c1/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java
b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java
index e1e230f..7c84c16 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java
@@ -30,6 +30,7 @@ import org.apache.solr.cloud.SolrCloudTestCase;
 import org.apache.solr.cloud.autoscaling.sim.SimCloudManager;
 import org.apache.solr.common.params.CollectionAdminParams;
 import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.util.Pair;
 import org.apache.solr.common.util.TimeSource;
 import org.apache.solr.core.SolrInfoBean;
 import org.apache.solr.metrics.SolrMetricManager;
@@ -57,7 +58,7 @@ public class MetricsHistoryHandlerTest extends SolrCloudTestCase {
 
   @BeforeClass
   public static void beforeClass() throws Exception {
-    simulated = random().nextBoolean();
+    simulated = random().nextBoolean() || true;
     Map<String, Object> args = new HashMap<>();
     args.put(MetricsHistoryHandler.SYNC_PERIOD_PROP, 1);
     args.put(MetricsHistoryHandler.COLLECT_PERIOD_PROP, 1);
@@ -111,11 +112,11 @@ public class MetricsHistoryHandlerTest extends SolrCloudTestCase {
   @Test
   public void testBasic() throws Exception {
     timeSource.sleep(10000);
-    List<String> list = handler.getFactory().list(100);
+    List<Pair<String, Long>> list = handler.getFactory().list(100);
     // solr.jvm, solr.node, solr.collection..system
     assertEquals(list.toString(), 3, list.size());
-    for (String path : list) {
-      RrdDb db = new RrdDb(MetricsHistoryHandler.URI_PREFIX + path, true, handler.getFactory());
+    for (Pair<String, Long> p : list) {
+      RrdDb db = new RrdDb(MetricsHistoryHandler.URI_PREFIX + p.first(), true, handler.getFactory());
       int dsCount = db.getDsCount();
       int arcCount = db.getArcCount();
       assertTrue("dsCount should be > 0, was " + dsCount, dsCount > 0);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2fa260c1/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
b/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
index 2f5fa13..18c72ec 100644
--- a/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
@@ -17,13 +17,13 @@
 
 package org.apache.solr.metrics.rrd;
 
-import java.util.Date;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.solr.SolrTestCaseJ4;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.params.CollectionAdminParams;
+import org.apache.solr.common.util.Pair;
 import org.apache.solr.common.util.TimeSource;
 import org.apache.solr.util.MockSearchableSolrClient;
 import org.junit.After;
@@ -78,15 +78,15 @@ public class SolrRrdBackendFactoryTest extends SolrTestCaseJ4 {
   @Test
   public void testBasic() throws Exception {
     RrdDb db = new RrdDb(createDef(), factory);
-    List<String> list = factory.list(100);
+    List<Pair<String, Long>> list = factory.list(100);
     assertEquals(list.toString(), 1, list.size());
-    assertEquals(list.toString(), "foo", list.get(0));
+    assertEquals(list.toString(), "foo", list.get(0).first());
     timeSource.sleep(2000);
     // there should be one sync data
     assertEquals(solrClient.docs.toString(), 1, solrClient.docs.size());
     String id = SolrRrdBackendFactory.ID_PREFIX + SolrRrdBackendFactory.ID_SEP + "foo";
     SolrInputDocument doc = solrClient.docs.get(CollectionAdminParams.SYSTEM_COLL).get(id);
-    long timestamp = ((Date)doc.getFieldValue("timestamp")).getTime();
+    long timestamp = (Long)doc.getFieldValue("timestamp_l");
     timeSource.sleep(2000);
     SolrInputDocument newDoc = solrClient.docs.get(CollectionAdminParams.SYSTEM_COLL).get(id);
     assertEquals(newDoc.toString(), newDoc, doc);
@@ -104,7 +104,7 @@ public class SolrRrdBackendFactoryTest extends SolrTestCaseJ4 {
     timeSource.sleep(3000);
     newDoc = solrClient.docs.get(CollectionAdminParams.SYSTEM_COLL).get(id);
     assertFalse(newDoc.toString(), newDoc.equals(doc));
-    long newTimestamp = ((Date)newDoc.getFieldValue("timestamp")).getTime();
+    long newTimestamp = (Long)newDoc.getFieldValue("timestamp_l");
     assertNotSame(newTimestamp, timestamp);
     FetchRequest fr = db.createFetchRequest(ConsolFun.AVERAGE, firstTimestamp + 60, lastTimestamp
- 60, 60);
     FetchData fd = fr.fetchData();
@@ -126,7 +126,7 @@ public class SolrRrdBackendFactoryTest extends SolrTestCaseJ4 {
     // should still be listed
     list = factory.list(100);
     assertEquals(list.toString(), 1, list.size());
-    assertEquals(list.toString(), "foo", list.get(0));
+    assertEquals(list.toString(), "foo", list.get(0).first());
 
     // re-open read-write
     db = new RrdDb("solr:foo", factory);
@@ -141,7 +141,7 @@ public class SolrRrdBackendFactoryTest extends SolrTestCaseJ4 {
     doc = newDoc;
     newDoc = solrClient.docs.get(CollectionAdminParams.SYSTEM_COLL).get(id);
     assertFalse(newDoc.toString(), newDoc.equals(doc));
-    newTimestamp = ((Date)newDoc.getFieldValue("timestamp")).getTime();
+    newTimestamp = (Long)newDoc.getFieldValue("timestamp_l");
     assertNotSame(newTimestamp, timestamp);
     fr = db.createFetchRequest(ConsolFun.AVERAGE, firstTimestamp + 60, lastTimestamp, 60);
     fd = fr.fetchData();
@@ -174,7 +174,7 @@ public class SolrRrdBackendFactoryTest extends SolrTestCaseJ4 {
     timestamp = newTimestamp;
     newDoc = solrClient.docs.get(CollectionAdminParams.SYSTEM_COLL).get(id);
     assertTrue(newDoc.toString(), newDoc.equals(doc));
-    newTimestamp = ((Date)newDoc.getFieldValue("timestamp")).getTime();
+    newTimestamp = (Long)newDoc.getFieldValue("timestamp_l");
     assertEquals(newTimestamp, timestamp);
     readOnly.close();
   }


Mime
View raw message