accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject [1/2] git commit: ACCUMULO-2645 add (very) basic data about running scans to identify run-away queries
Date Thu, 11 Sep 2014 19:21:57 GMT
Repository: accumulo
Updated Branches:
  refs/heads/master 6104c98b9 -> e3b0412e1


ACCUMULO-2645 add (very) basic data about running scans to identify run-away queries


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7f329a9c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7f329a9c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7f329a9c

Branch: refs/heads/master
Commit: 7f329a9c4dbf6f249a14a1f4bde43f261bf12875
Parents: 560c268
Author: Eric C. Newton <eric.newton@gmail.com>
Authored: Thu Sep 11 15:17:39 2014 -0400
Committer: Eric C. Newton <eric.newton@gmail.com>
Committed: Thu Sep 11 15:17:39 2014 -0400

----------------------------------------------------------------------
 .../org/apache/accumulo/monitor/Monitor.java    | 65 +++++++++++++++++-
 .../accumulo/monitor/servlets/BasicServlet.java |  1 +
 .../accumulo/monitor/servlets/ScanServlet.java  | 72 ++++++++++++++++++++
 3 files changed, 136 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/7f329a9c/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
----------------------------------------------------------------------
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 0e6b37f..b0124a9 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@ -21,13 +21,18 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.TreeMap;
 
+import org.apache.accumulo.monitor.servlets.ScanServlet;
+import com.google.common.net.HostAndPort;
 import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.core.client.impl.MasterClient;
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
@@ -39,6 +44,8 @@ import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
 import org.apache.accumulo.core.master.thrift.TableInfo;
 import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
 import org.apache.accumulo.core.util.Daemon;
 import org.apache.accumulo.core.util.LoggingRunnable;
 import org.apache.accumulo.core.util.Pair;
@@ -83,8 +90,6 @@ import org.apache.accumulo.trace.instrument.Tracer;
 import org.apache.log4j.Logger;
 import org.apache.zookeeper.KeeperException;
 
-import com.google.common.net.HostAndPort;
-
 /**
  * Serve master statistics with an embedded web server.
  */
@@ -436,6 +441,7 @@ public class Monitor {
     server.addServlet(XMLServlet.class, "/xml");
     server.addServlet(JSONServlet.class, "/json");
     server.addServlet(VisServlet.class, "/vis");
+    server.addServlet(ScanServlet.class, "/scans");
     server.addServlet(Summary.class, "/trace/summary");
     server.addServlet(ListType.class, "/trace/listType");
     server.addServlet(ShowTrace.class, "/trace/show");
@@ -482,6 +488,61 @@ public class Monitor {
 
       }
     }), "Data fetcher").start();
+    
+    new Daemon(new LoggingRunnable(log, new Runnable() {
+      @Override
+      public void run() {
+        while (true) {
+          try {
+            Monitor.fetchScans();
+          } catch (Exception e) {
+            log.warn(e.getMessage(), e);
+          }
+          UtilWaitThread.sleep(5000);
+        }
+      }
+    }), "Scan scanner").start();
+  }
+  
+  public static class ScanStats {
+    public final List<ActiveScan> scans;
+    public final long fetched;
+    ScanStats(List<ActiveScan> active) {
+      this.scans = active;
+      this.fetched = System.currentTimeMillis();
+    }
+  }
+  static final Map<String, ScanStats> allScans = new HashMap<String, ScanStats>();
+  public static Map<String, ScanStats> getScans() {
+    synchronized (allScans) {
+      return new TreeMap<String, ScanStats>(allScans);
+    }
+  }
+
+  protected static void fetchScans() throws Exception {
+    if (instance == null)
+      return;
+    Connector c = instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
+    for (String server : c.instanceOperations().getTabletServers()) {
+      Client tserver = ThriftUtil.getTServerClient(server, Monitor.getSystemConfiguration());
+      try {
+        List<ActiveScan> scans = tserver.getActiveScans(null, SystemCredentials.get().toThrift(instance));
+        synchronized (allScans) {
+          allScans.put(server, new ScanStats(scans));
+        }
+      } catch (Exception ex) {
+        log.debug(ex, ex);
+      }
+    }
+    // Age off old scan information
+    Iterator<Entry<String,ScanStats>> entryIter = allScans.entrySet().iterator();
+    long now = System.currentTimeMillis();
+    while (entryIter.hasNext()) {
+      Entry<String,ScanStats> entry = entryIter.next();
+      if (now - entry.getValue().fetched > 5 * 60 * 1000) {
+        entryIter.remove();
+      }
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/accumulo/blob/7f329a9c/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
----------------------------------------------------------------------
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
index 603f55f..d896040 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/BasicServlet.java
@@ -184,6 +184,7 @@ abstract public class BasicServlet extends HttpServlet {
     sb.append("<hr />\n");
     sb.append("<a href='/master'>Master&nbsp;Server</a><br />\n");
     sb.append("<a href='/tservers'>Tablet&nbsp;Servers</a><br />\n");
+    sb.append("<a href='/scans'>Active&nbsp;Scans</a><br />\n");
     sb.append("<a href='/vis'>Server Activity</a><br />\n");
     sb.append("<a href='/gc'>Garbage&nbsp;Collector</a><br />\n");
     sb.append("<a href='/tables'>Tables</a><br />\n");

http://git-wip-us.apache.org/repos/asf/accumulo/blob/7f329a9c/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ScanServlet.java
----------------------------------------------------------------------
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ScanServlet.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ScanServlet.java
new file mode 100644
index 0000000..12b0871
--- /dev/null
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ScanServlet.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor.servlets;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+
+import org.apache.accumulo.monitor.util.celltypes.TServerLinkType;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
+import org.apache.accumulo.monitor.Monitor;
+import org.apache.accumulo.monitor.Monitor.ScanStats;
+import org.apache.accumulo.monitor.util.Table;
+import org.apache.accumulo.monitor.util.TableRow;
+import org.apache.accumulo.monitor.util.celltypes.DurationType;
+import org.apache.accumulo.monitor.util.celltypes.StringType;
+
+public class ScanServlet extends BasicServlet {
+
+  private static final long serialVersionUID = 1L;
+
+  @Override
+  protected String getTitle(HttpServletRequest req) {
+    return "Scans";
+  }
+  
+  @Override
+  protected void pageBody(HttpServletRequest req, HttpServletResponse response, StringBuilder
sb) throws IOException {
+    Map<String,ScanStats> scans = Monitor.getScans();
+    Table scanTable = new Table("scanStatus", "Scan&nbsp;Status");
+    scanTable.addSortableColumn("Server", new TServerLinkType(), null);
+    scanTable.addSortableColumn("#", new PreciseNumberType(0, 20, 0, 100), "Number of scans
presently running");
+    scanTable.addSortableColumn("Oldest&nbsp;Age", new DurationType(0l, 5 * 60 * 100l),
"The age of the oldest scan on this server.");
+    for (TabletServerStatus tserverInfo : Monitor.getMmi().getTServerInfo()) {
+      ScanStats stats = scans.get(tserverInfo.name);
+      long count = 0;
+      long oldest = 0;
+      if (stats != null) {
+        count = stats.scans.size();
+        for (ActiveScan scan : stats.scans) {
+          oldest = Math.max(scan.age, oldest);
+        }
+      }
+      TableRow row = scanTable.prepareRow();
+      row.add(tserverInfo);
+      row.add(count);
+      row.add(oldest);
+      scanTable.addRow(row);
+    }
+    scanTable.generate(req, sb);
+  }
+  
+}


Mime
View raw message