incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [2/2] git commit: More blur shell top updates.
Date Mon, 24 Jun 2013 14:50:08 GMT
More blur shell top updates.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/d54a7fc8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/d54a7fc8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/d54a7fc8

Branch: refs/heads/master
Commit: d54a7fc81ef1a5c956b2a20f943c49d457589fb7
Parents: c382549
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Mon Jun 24 10:49:37 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Mon Jun 24 10:49:37 2013 -0400

----------------------------------------------------------------------
 .../blur/server/ShardServerEventHandler.java    |  26 +++
 .../java/org/apache/blur/shell/TopCommand.java  | 233 ++++++++++++-------
 .../org/apache/blur/shell/top.properties        |  66 ++++++
 3 files changed, 244 insertions(+), 81 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d54a7fc8/blur-core/src/main/java/org/apache/blur/server/ShardServerEventHandler.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/ShardServerEventHandler.java b/blur-core/src/main/java/org/apache/blur/server/ShardServerEventHandler.java
index d8eb7cb..0f0742a 100644
--- a/blur-core/src/main/java/org/apache/blur/server/ShardServerEventHandler.java
+++ b/blur-core/src/main/java/org/apache/blur/server/ShardServerEventHandler.java
@@ -16,6 +16,12 @@ package org.apache.blur.server;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import static org.apache.blur.metrics.MetricsConstants.BLUR;
+import static org.apache.blur.metrics.MetricsConstants.ORG_APACHE_BLUR;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocol;
@@ -23,6 +29,11 @@ import org.apache.blur.thirdparty.thrift_0_9_0.server.ServerContext;
 import org.apache.blur.thirdparty.thrift_0_9_0.server.TServerEventHandler;
 import org.apache.blur.thirdparty.thrift_0_9_0.transport.TTransport;
 
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Gauge;
+import com.yammer.metrics.core.Meter;
+import com.yammer.metrics.core.MetricName;
+
 /**
  * {@link ShardServerContext} is the session manager for the shard servers. It
  * allows for reader reuse across method calls.
@@ -30,6 +41,18 @@ import org.apache.blur.thirdparty.thrift_0_9_0.transport.TTransport;
 public class ShardServerEventHandler implements TServerEventHandler {
 
   private static final Log LOG = LogFactory.getLog(ShardServerEventHandler.class);
+  private final Meter _connectionMeter;
+  private final AtomicLong _connections = new AtomicLong();
+  
+  public ShardServerEventHandler() {
+    Metrics.newGauge(new MetricName(ORG_APACHE_BLUR, BLUR, "Connections"), new Gauge<Long>()
{
+      @Override
+      public Long value() {
+        return null;
+      }
+    });
+    _connectionMeter = Metrics.newMeter(new MetricName(ORG_APACHE_BLUR, BLUR, "Connections/s"),
"Connections/s", TimeUnit.SECONDS);
+  }
 
   @Override
   public void preServe() {
@@ -39,6 +62,8 @@ public class ShardServerEventHandler implements TServerEventHandler {
   @Override
   public ServerContext createContext(TProtocol input, TProtocol output) {
     LOG.debug("Client connected");
+    _connectionMeter.mark();
+    _connections.incrementAndGet();
     return new ShardServerContext();
   }
 
@@ -47,6 +72,7 @@ public class ShardServerEventHandler implements TServerEventHandler {
     LOG.debug("Client disconnected");
     ShardServerContext context = (ShardServerContext) serverContext;
     context.close();
+    _connections.decrementAndGet();
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d54a7fc8/blur-shell/src/main/java/org/apache/blur/shell/TopCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/TopCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/TopCommand.java
index b712c07..ca6eee8 100644
--- a/blur-shell/src/main/java/org/apache/blur/shell/TopCommand.java
+++ b/blur-shell/src/main/java/org/apache/blur/shell/TopCommand.java
@@ -20,11 +20,15 @@ package org.apache.blur.shell;
 
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Properties;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
@@ -35,54 +39,26 @@ import jline.Terminal;
 import jline.console.ConsoleReader;
 
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thirdparty.thrift_0_9_0.transport.TTransportException;
 import org.apache.blur.thrift.BlurClientManager;
 import org.apache.blur.thrift.Connection;
 import org.apache.blur.thrift.generated.Blur;
 import org.apache.blur.thrift.generated.Blur.Client;
-import org.apache.blur.thrift.generated.Blur.Iface;
 import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.Metric;
 
 public class TopCommand extends Command {
 
+  private static final String TOP = "top.";
+  private static final String LONGNAME = ".longname";
+  private static final String SHORTNAME = ".shortname";
+  private static final String UNKNOWN = "Unknown";
+  private static final String TOP_SHARD_SERVER_SHORTNAME = "top.SHARD_SERVER.shortname";
+
   public enum SCREEN {
     HELP, TOP
   }
 
-  private static final String SC = "seg cnt";
-  private static final String IC = "idx cnt";
-  private static final String TC = "tble cnt";
-  private static final String RE = "rd rec";
-  private static final String IM = "idx mem";
-  private static final String RO = "rd row";
-  private static final String CS = "bc size";
-  private static final String CE = "bc evt";
-  private static final String CM = "bc miss";
-  private static final String CH = "bc hit";
-  private static final String IQ = "in qry";
-  private static final String EQ = "ex qry";
-  private static final String HU = "heap usd";
-  private static final String SL = "sys load";
-
-  private static final String SHARD_SERVER = "Shard Server";
-  private static final String INTERNAL_QUERIES = "\"org.apache.blur\":type=\"Blur\",name=\"Internal
Queries/s\"";
-  private static final String EXTERNAL_QUERIES = "\"org.apache.blur\":type=\"Blur\",name=\"External
Queries/s\"";
-
-  private static final String CACHE_HIT = "\"org.apache.blur\":type=\"Cache\",name=\"Hit\"";
-  private static final String CACHE_MISS = "\"org.apache.blur\":type=\"Cache\",name=\"Miss\"";
-  private static final String CACHE_EVICTION = "\"org.apache.blur\":type=\"Cache\",name=\"Eviction\"";
-  private static final String CACHE_SIZE = "\"org.apache.blur\":type=\"Cache\",name=\"Size\"";
-
-  private static final String READ_RECORDS = "\"org.apache.blur\":type=\"Blur\",name=\"Read
Records/s\"";
-  private static final String READ_ROWS = "\"org.apache.blur\":type=\"Blur\",name=\"Read
Row/s\"";
-
-  private static final String INDEX_MEMORY_USAGE = "\"org.apache.blur\":type=\"Blur\",scope=\"default\",name=\"Index
Memory Usage\"";
-  private static final String TABLE_COUNT = "\"org.apache.blur\":type=\"Blur\",scope=\"default\",name=\"Table
Count\"";
-  private static final String INDEX_COUNT = "\"org.apache.blur\":type=\"Blur\",scope=\"default\",name=\"Index
Count\"";
-  private static final String SEGMENT_COUNT = "\"org.apache.blur\":type=\"Blur\",scope=\"default\",name=\"Segment
Count\"";
-  private static final String LOAD_AVERAGE = "\"org.apache.blur\":type=\"System\",name=\"Load
Average\"";
-  private static final String HEAP_USED = "\"org.apache.blur\":type=\"JVM\",name=\"Heap Used\"";
-
   private static final double ONE_THOUSAND = 1000;
   private static final double ONE_MILLION = ONE_THOUSAND * ONE_THOUSAND;
   private static final double ONE_BILLION = ONE_THOUSAND * ONE_MILLION;
@@ -111,28 +87,32 @@ public class TopCommand extends Command {
     AtomicBoolean quit = new AtomicBoolean();
     AtomicBoolean help = new AtomicBoolean();
 
+    Properties properties = new Properties();
+    try {
+      properties.load(getClass().getResourceAsStream("top.properties"));
+    } catch (IOException e) {
+      if (Main.debug) {
+        e.printStackTrace();
+      }
+      throw new CommandException(e.getMessage());
+    }
+
     Map<String, String> metricNames = new HashMap<String, String>();
-    metricNames.put(IQ, INTERNAL_QUERIES);
-    metricNames.put(EQ, EXTERNAL_QUERIES);
-    metricNames.put(CH, CACHE_HIT);
-    metricNames.put(CM, CACHE_MISS);
-    metricNames.put(CE, CACHE_EVICTION);
-    metricNames.put(CS, CACHE_SIZE);
-    metricNames.put(RO, READ_RECORDS);
-    metricNames.put(RE, READ_ROWS);
-    metricNames.put(IM, INDEX_MEMORY_USAGE);
-    metricNames.put(TC, TABLE_COUNT);
-    metricNames.put(IC, INDEX_COUNT);
-    metricNames.put(SC, SEGMENT_COUNT);
-    metricNames.put(HU, HEAP_USED);
-    metricNames.put(SL, LOAD_AVERAGE);
-
-    Object[] labels = new Object[] { SHARD_SERVER, SL, HU, IM, EQ, IQ, RO, RE, CH, CM, CE,
CS, TC, IC, SC };
-
-    Set<String> sizes = new HashSet<String>();
-    sizes.add(IM);
-    sizes.add(SL);
+    Set<Object> keySet = properties.keySet();
+    for (Object k : keySet) {
+      String key = k.toString();
+      if (isShortName(key)) {
+        String shortName = getShortName(key, properties);
+        String longName = getLongName(getLongNameKey(key), properties);
+        metricNames.put(shortName, longName);
+      }
+    }
+
+    String labelsStr = properties.getProperty("top.columns");
+    String[] labels = resolveShortNames(labelsStr.split(","), properties);
 
+    String sizesStr = properties.getProperty("top.sizes");
+    Set<String> sizes = new HashSet<String>(Arrays.asList(resolveShortNames(sizesStr.split(","),
properties)));
     Set<String> keys = new HashSet<String>(metricNames.values());
 
     String cluster;
@@ -142,8 +122,6 @@ public class TopCommand extends Command {
       cluster = args[1];
     }
 
-    List<String> shardServerList = client.shardServerList(cluster);
-
     ConsoleReader reader = this.getConsoleReader();
     if (reader != null) {
       Terminal terminal = reader.getTerminal();
@@ -161,25 +139,12 @@ public class TopCommand extends Command {
       startCommandWatcher(reader, quit, help, this);
     }
 
-    Map<String, AtomicReference<Blur.Iface>> shardClients = new ConcurrentHashMap<String,
AtomicReference<Blur.Iface>>();
-    for (String sc : shardServerList) {
-      AtomicReference<Iface> ref = shardClients.get(sc);
-      if (ref == null) {
-        ref = new AtomicReference<Blur.Iface>();
-        shardClients.put(sc, ref);
-      }
-      try {
-        Client c = BlurClientManager.newClient(new Connection(sc));
-        ref.set(c);
-      } catch (IOException e) {
-        ref.set(null);
-        if (Main.debug) {
-          e.printStackTrace();
-        }
-      }
-    }
+    List<String> shardServerList = new ArrayList<String>(client.shardServerList(cluster));
+    Collections.sort(shardServerList);
+    Map<String, AtomicReference<Client>> shardClients = setupClients(shardServerList);
 
-    int longestServerName = Math.max(getSizeOfLongestKey(shardClients), SHARD_SERVER.length());
+    String shardServerLabel = properties.getProperty(TOP_SHARD_SERVER_SHORTNAME);
+    int longestServerName = Math.max(getSizeOfLongestKey(shardClients), shardServerLabel.length());
 
     StringBuilder header = new StringBuilder("%" + longestServerName + "s");
     for (int i = 1; i < labels.length; i++) {
@@ -194,12 +159,13 @@ public class TopCommand extends Command {
       } else if (help.get()) {
         showHelp(output, labels, metricNames);
       } else {
-        output.append(truncate(String.format(header.toString(), labels)));
-        for (Entry<String, AtomicReference<Blur.Iface>> e : new TreeMap<String,
AtomicReference<Blur.Iface>>(
-            shardClients).entrySet()) {
+        output.append(truncate(String.format(header.toString(), (Object[]) labels)));
+        for (Entry<String, AtomicReference<Client>> e : new TreeMap<String,
AtomicReference<Client>>(shardClients)
+            .entrySet()) {
           String shardServer = e.getKey();
-          Iface shardClient = e.getValue().get();
-          if (shardClient == null) {
+          AtomicReference<Client> ref = e.getValue();
+          Map<String, Metric> metrics = getMetrics(shardServer, ref, keys);
+          if (metrics == null) {
             String line = String.format("%" + longestServerName + "s*%n", shardServer);
             output.append(line);
           } else {
@@ -207,7 +173,7 @@ public class TopCommand extends Command {
             int c = 0;
             cols[c++] = shardServer;
             StringBuilder sb = new StringBuilder("%" + longestServerName + "s");
-            Map<String, Metric> metrics = shardClient.metrics(keys);
+
             for (int i = 1; i < labels.length; i++) {
               String mn = metricNames.get(labels[i]);
               Metric metric = metrics.get(mn);
@@ -244,9 +210,114 @@ public class TopCommand extends Command {
         Terminal terminal = reader.getTerminal();
         _height = terminal.getHeight() - 2;
         _width = terminal.getWidth() - 2;
+        
+        List<String> currentShardServerList = new ArrayList<String>(client.shardServerList(cluster));
+        Collections.sort(currentShardServerList);
+        if (!shardServerList.equals(currentShardServerList)) {
+          close(shardClients);
+          shardClients = setupClients(shardServerList);
+        }
       }
     } while (reader != null);
+  }
+
+  private void close(Map<String, AtomicReference<Client>> shardClients) {
+    for (AtomicReference<Client> client : shardClients.values()) {
+      tryToClose(client);
+    }
+  }
+
+  private Map<String, AtomicReference<Client>> setupClients(List<String>
shardServerList) {
+    Map<String, AtomicReference<Client>> shardClients = new ConcurrentHashMap<String,
AtomicReference<Client>>();
+    for (String sc : shardServerList) {
+      AtomicReference<Client> ref = shardClients.get(sc);
+      if (ref == null) {
+        ref = new AtomicReference<Client>();
+        shardClients.put(sc, ref);
+      }
+      tryToConnect(sc, ref);
+    }
+    return shardClients;
+  }
+
+  private boolean tryToConnect(String sc, AtomicReference<Client> ref) {
+    try {
+      Client c = BlurClientManager.newClient(new Connection(sc));
+      ref.set(c);
+      return true;
+    } catch (TTransportException e) {
+      ref.set(null);
+      if (Main.debug) {
+        e.printStackTrace();
+      }
+    } catch (IOException e) {
+      ref.set(null);
+      if (Main.debug) {
+        e.printStackTrace();
+      }
+    }
+    return false;
+  }
+
+  private Map<String, Metric> getMetrics(String sc, AtomicReference<Client> ref,
Set<String> keys) {
+    if (ref.get() == null) {
+      if (!tryToConnect(sc, ref)) {
+        return null;
+      }
+    }
+    try {
+      return ref.get().metrics(keys);
+    } catch (BlurException e) {
+      if (Main.debug) {
+        e.printStackTrace();
+      }
+    } catch (TException e) {
+      if (Main.debug) {
+        e.printStackTrace();
+      }
+      tryToClose(ref);
+    }
+    return null;
+  }
+
+  private void tryToClose(AtomicReference<Client> ref) {
+    Client client = ref.get();
+    if (client != null) {
+      ref.set(null);
+      try {
+        client.getInputProtocol().getTransport().close();
+      } catch (Exception e) {
+        if (Main.debug) {
+          e.printStackTrace();
+        }
+      }
+    }
+  }
+
+  private String[] resolveShortNames(String[] keys, Properties properties) {
+    String[] labels = new String[keys.length];
+    int i = 0;
+    for (String key : keys) {
+      String shortName = properties.getProperty(TOP + key + SHORTNAME);
+      labels[i++] = shortName;
+    }
+    return labels;
+  }
+
+  private String getLongNameKey(String key) {
+    return key.replace(SHORTNAME, LONGNAME);
+  }
+
+  private String getShortName(String key, Properties properties) {
+    return properties.getProperty(key, UNKNOWN);
+  }
+
+  private String getLongName(String key, Properties properties) {
+    return properties.getProperty(key, UNKNOWN);
+  }
 
+  private boolean isShortName(String key) {
+    return key.endsWith(SHORTNAME);
   }
 
   private void showHelp(StringBuilder output, Object[] labels, Map<String, String>
metricNames) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d54a7fc8/blur-shell/src/main/resources/org/apache/blur/shell/top.properties
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/resources/org/apache/blur/shell/top.properties b/blur-shell/src/main/resources/org/apache/blur/shell/top.properties
new file mode 100644
index 0000000..9f3cbe2
--- /dev/null
+++ b/blur-shell/src/main/resources/org/apache/blur/shell/top.properties
@@ -0,0 +1,66 @@
+top.columns=SHARD_SERVER,SL,HU,IM,EQ,IQ,RO,RE,CH,CM,CE,CS,TC,IC,SC
+top.sizes=IM,SL
+
+top.SHARD_SERVER.shortname=Shard Server
+top.SC.shortname=seg cnt
+top.IC.shortname=idx cnt
+top.TC.shortname=tble cnt
+top.RE.shortname=rd rec
+top.IM.shortname=idx mem
+top.RO.shortname=rd row
+top.CS.shortname=bc size
+top.CE.shortname=bc evt
+top.CM.shortname=bc miss
+top.CH.shortname=bc hit
+top.IQ.shortname=in qry
+top.EQ.shortname=ex qry
+top.HU.shortname=heap usd
+top.SL.shortname=sys load
+
+top.SHARD_SERVER.longname=Shard Server
+top.IQ.longname="org.apache.blur":type="Blur",name="Internal Queries/s"
+top.EQ.longname="org.apache.blur":type="Blur",name="External Queries/s"
+top.CH.longname="org.apache.blur":type="Cache",name="Hit"
+top.CM.longname="org.apache.blur":type="Cache",name="Miss"
+top.CE.longname="org.apache.blur":type="Cache",name="Eviction"
+top.CS.longname="org.apache.blur":type="Cache",name="Size"
+top.RE.longname="org.apache.blur":type="Blur",name="Read Records/s"
+top.RO.longname="org.apache.blur":type="Blur",name="Read Row/s"
+top.IM.longname="org.apache.blur":type="Blur",scope="default",name="Index Memory Usage"
+top.TC.longname="org.apache.blur":type="Blur",scope="default",name="Table Count"
+top.IC.longname="org.apache.blur":type="Blur",scope="default",name="Index Count"
+top.SC.longname="org.apache.blur":type="Blur",scope="default",name="Segment Count"
+top.SL.longname="org.apache.blur":type="System",name="Load Average"
+top.HU.longname="org.apache.blur":type="JVM",name="Heap Used"
+
+top.SHARD_SERVER.longname=Shard Server
+top.IQ.longname="org.apache.blur":type="Blur",name="Internal Queries/s"
+top.EQ.longname="org.apache.blur":type="Blur",name="External Queries/s"
+top.CH.longname="org.apache.blur":type="Cache",name="Hit"
+top.CM.longname="org.apache.blur":type="Cache",name="Miss"
+top.CE.longname="org.apache.blur":type="Cache",name="Eviction"
+top.CS.longname="org.apache.blur":type="Cache",name="Size"
+top.RE.longname="org.apache.blur":type="Blur",name="Read Records/s"
+top.RO.longname="org.apache.blur":type="Blur",name="Read Row/s"
+top.IM.longname="org.apache.blur":type="Blur",scope="default",name="Index Memory Usage"
+top.TC.longname="org.apache.blur":type="Blur",scope="default",name="Table Count"
+top.IC.longname="org.apache.blur":type="Blur",scope="default",name="Index Count"
+top.SC.longname="org.apache.blur":type="Blur",scope="default",name="Segment Count"
+top.SL.longname="org.apache.blur":type="System",name="Load Average"
+top.HU.longname="org.apache.blur":type="JVM",name="Heap Used"
+
+top.SHARD_SERVER.help=Shard Server - * indicates that the server is offline
+top.IQ.help=Internal Queries/s - This counts every shard query (One Minute Average)
+top.EQ.help=External Queries/s - This counts every query request no matter the number of
shards (One Minute Average)
+top.CH.help=Block Cache Hits/s - The number of 8k block hits (One Minute Average)
+top.CM.help=Block Cache Misses/s - The number of 8k block misses (One Minute Average)
+top.CE.help=Block Cache Evictions/s - The number of 8k block evictions (One Minute Average)
+top.CS.help=Block Cache Size - The total block size
+top.RE.help=Read Records/s - The number of records read per second (One Minute Average)
+top.RO.help=Read Row/s - The number of rows read per second (One Minute Average)
+top.IM.help=Index Memory Usage - The index heap usage
+top.TC.help=Table Count - The number of tables
+top.IC.help=Index Count - The number of indexes
+top.SC.help=Segment Count - The number of index segments 
+top.SL.help=Load Average - Current server load average
+top.HU.help=Heap Used - Current heap size
\ No newline at end of file


Mime
View raw message