incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [6/7] git commit: Inital commit of the top command in the blur-shell.
Date Sat, 22 Jun 2013 21:07:07 GMT
Inital commit of the top command in the blur-shell.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/7137006b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/7137006b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/7137006b

Branch: refs/heads/master
Commit: 7137006b58f1b55d1d36f1386333a6122ed11beb
Parents: 878381e
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Sat Jun 22 17:04:22 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Sat Jun 22 17:04:22 2013 -0400

----------------------------------------------------------------------
 .../org/apache/blur/manager/IndexManager.java   |   12 +
 .../blur/thrift/BlurControllerServer.java       |   28 +
 .../org/apache/blur/thrift/BlurShardServer.java |   13 +-
 .../java/org/apache/blur/thrift/TableAdmin.java |   26 +
 .../blur/thrift/ThriftBlurControllerServer.java |    2 +
 .../blur/thrift/ThriftBlurShardServer.java      |    2 +
 .../org/apache/blur/utils/MemoryReporter.java   |  159 +
 .../main/java/org/apache/blur/shell/Main.java   |   14 +-
 .../org/apache/blur/shell/ParseCommand.java     |   51 +
 .../java/org/apache/blur/shell/TopCommand.java  |  277 +
 .../org/apache/blur/thrift/generated/Blur.java  | 8248 +++++++++++-------
 .../apache/blur/thrift/generated/Metric.java    |  894 ++
 .../apache/blur/thrift/util/QueryMetrics.java   |   43 +
 .../src/main/scripts/interface/Blur.thrift      |   17 +-
 .../main/scripts/interface/gen-html/Blur.html   |   21 +-
 .../main/scripts/interface/gen-html/index.html  |    3 +
 .../org/apache/blur/thrift/generated/Blur.java  | 8248 +++++++++++-------
 .../apache/blur/thrift/generated/Metric.java    |  894 ++
 .../src/main/scripts/interface/gen-js/Blur.js   |  808 +-
 .../main/scripts/interface/gen-js/Blur_types.js |  194 +
 .../scripts/interface/gen-perl/Blur/Blur.pm     |  780 +-
 .../scripts/interface/gen-perl/Blur/Types.pm    |  184 +
 .../src/main/scripts/interface/gen-rb/blur.rb   |  124 +
 .../main/scripts/interface/gen-rb/blur_types.rb |   22 +
 pom.xml                                         |    2 +-
 25 files changed, 14494 insertions(+), 6572 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
index 1063b7f..7280ca7 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
@@ -406,6 +406,18 @@ public class IndexManager {
     }
   }
 
+  public String parseQuery(String table, SimpleQuery simpleQuery) throws ParseException,
BlurException {
+    TableContext context = getTableContext(table);
+    BlurAnalyzer analyzer = _indexServer.getAnalyzer(table);
+    Filter preFilter = QueryParserUtil.parseFilter(table, simpleQuery.preSuperFilter, false,
analyzer, _filterCache,
+        context);
+    Filter postFilter = QueryParserUtil.parseFilter(table, simpleQuery.postSuperFilter, true,
analyzer, _filterCache,
+        context);
+    Query userQuery = QueryParserUtil.parseQuery(simpleQuery.queryStr, simpleQuery.superQueryOn,
analyzer, postFilter,
+        preFilter, getScoreType(simpleQuery.type), context);
+    return userQuery.toString();
+  }
+
   private TableContext getTableContext(final String table) {
     return TableContext.create(_clusterStatus.getTableDescriptor(true, _clusterStatus.getCluster(true,
table), table));
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
index ecdccc7..54524f4 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
@@ -69,6 +69,7 @@ import org.apache.blur.thrift.generated.RowMutation;
 import org.apache.blur.thrift.generated.Schema;
 import org.apache.blur.thrift.generated.Selector;
 import org.apache.blur.thrift.generated.ShardState;
+import org.apache.blur.thrift.generated.SimpleQuery;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.blur.thrift.generated.TableStats;
 import org.apache.blur.utils.BlurConstants;
@@ -911,4 +912,31 @@ public class BlurControllerServer extends TableAdmin implements Iface
{
     }
   }
 
+  @Override
+  public String parseQuery(final String table, final SimpleQuery simpleQuery) throws BlurException,
TException {
+    checkTable(table);
+    String cluster = getCluster(table);
+    List<String> onlineShardServers = _clusterStatus.getOnlineShardServers(true, cluster);
+    try {
+      return BlurClientManager.execute(getConnections(onlineShardServers), new BlurCommand<String>()
{
+        @Override
+        public String call(Client client) throws BlurException, TException {
+          return client.parseQuery(table, simpleQuery);
+        }
+      });
+    } catch (Exception e) {
+      LOG.error("Unknown error while trying to parse query [table={0},simpleQuery={1}]",
e, table, simpleQuery);
+      throw new BException("Unknown error while trying to parse query [table={0},simpleQuery={1}]",
e, table,
+          simpleQuery);
+    }
+  }
+
+  private List<Connection> getConnections(List<String> onlineShardServers) {
+    List<Connection> connections = new ArrayList<Connection>();
+    for (String c : onlineShardServers) {
+      connections.add(new Connection(c));
+    }
+    return connections;
+  }
+
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
index db2a817..2d6a022 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
@@ -52,6 +52,7 @@ import org.apache.blur.thrift.generated.RowMutation;
 import org.apache.blur.thrift.generated.Schema;
 import org.apache.blur.thrift.generated.Selector;
 import org.apache.blur.thrift.generated.ShardState;
+import org.apache.blur.thrift.generated.SimpleQuery;
 import org.apache.blur.thrift.generated.TableStats;
 import org.apache.blur.utils.BlurConstants;
 import org.apache.blur.utils.BlurUtil;
@@ -100,7 +101,7 @@ public class BlurShardServer extends TableAdmin implements Iface {
           highlightOptions.setSimpleQuery(blurQuery.getSimpleQuery());
         }
       }
-      
+
       if (blurQuery.useCacheIfPresent) {
         LOG.debug("Using cache for query [{0}] on table [{1}].", blurQuery, table);
         QueryCacheKey key = QueryCache.getNormalizedBlurQueryKey(table, blurQuery);
@@ -134,6 +135,16 @@ public class BlurShardServer extends TableAdmin implements Iface {
   }
 
   @Override
+  public String parseQuery(String table, SimpleQuery simpleQuery) throws BlurException, TException
{
+    try {
+      return _indexManager.parseQuery(table, simpleQuery);
+    } catch (Throwable e) {
+      LOG.error("Unknown error during parsing of [table={0},simpleQuery={1}]", e, table,
simpleQuery);
+      throw new BException(e.getMessage(), e);
+    }
+  }
+
+  @Override
   public FetchResult fetchRow(String table, Selector selector) throws BlurException, TException
{
     checkTable(_cluster, table);
     try {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
index 8069764..5826e27 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
@@ -16,9 +16,11 @@ package org.apache.blur.thrift;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 
 import org.apache.blur.BlurConfiguration;
 import org.apache.blur.log.Log;
@@ -28,10 +30,12 @@ import org.apache.blur.server.TableContext;
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
 import org.apache.blur.thrift.generated.Blur.Iface;
 import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.Metric;
 import org.apache.blur.thrift.generated.ShardState;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.blur.thrift.generated.TableStats;
 import org.apache.blur.utils.BlurUtil;
+import org.apache.blur.utils.MemoryReporter;
 import org.apache.zookeeper.ZooKeeper;
 
 public abstract class TableAdmin implements Iface {
@@ -42,6 +46,28 @@ public abstract class TableAdmin implements Iface {
   protected BlurConfiguration _configuration;
 
   @Override
+  public Map<String, Metric> metrics(Set<String> metrics) throws BlurException,
TException {
+    try {
+      Map<String, Metric> metricsMap = MemoryReporter.getMetrics();
+      if (metrics == null) {
+        return metricsMap;
+      } else {
+        Map<String, Metric> result = new HashMap<String, Metric>();
+        for (String n : metrics) {
+          Metric metric = metricsMap.get(n);
+          if (metric != null) {
+            result.put(n, metric);
+          }
+        }
+        return result;
+      }
+    } catch (Exception e) {
+      LOG.error("Unknown error while trying to get metrics [{0}] ", e, metrics);
+      throw new BException(e.getMessage(), e);
+    }
+  }
+
+  @Override
   public TableStats getTableStats(String table) throws BlurException, TException {
     return tableStats(table);
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
index 5780d27..5a5c4a1 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurControllerServer.java
@@ -51,6 +51,7 @@ import org.apache.blur.metrics.ReporterSetup;
 import org.apache.blur.thrift.generated.Blur.Iface;
 import org.apache.blur.utils.BlurConstants;
 import org.apache.blur.utils.BlurUtil;
+import org.apache.blur.utils.MemoryReporter;
 import org.apache.blur.zookeeper.ZkUtils;
 import org.apache.zookeeper.ZooKeeper;
 
@@ -65,6 +66,7 @@ public class ThriftBlurControllerServer extends ThriftServer {
     BlurConfiguration configuration = new BlurConfiguration();
     printUlimits();
     ReporterSetup.setupReporters(configuration);
+    MemoryReporter.enable();
     ThriftServer server = createServer(serverIndex, configuration);
     server.start();
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index 45d0dd9..ff2cec6 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -71,6 +71,7 @@ import org.apache.blur.thirdparty.thrift_0_9_0.server.TServlet;
 import org.apache.blur.thrift.generated.Blur;
 import org.apache.blur.thrift.generated.Blur.Iface;
 import org.apache.blur.utils.BlurUtil;
+import org.apache.blur.utils.MemoryReporter;
 import org.apache.blur.zookeeper.ZkUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.zookeeper.ZooKeeper;
@@ -89,6 +90,7 @@ public class ThriftBlurShardServer extends ThriftServer {
     BlurConfiguration configuration = new BlurConfiguration();
     printUlimits();
     ReporterSetup.setupReporters(configuration);
+    MemoryReporter.enable();
     ThriftServer server = createServer(serverIndex, configuration);
     server.start();
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-core/src/main/java/org/apache/blur/utils/MemoryReporter.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/MemoryReporter.java b/blur-core/src/main/java/org/apache/blur/utils/MemoryReporter.java
new file mode 100644
index 0000000..a64f51b
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/utils/MemoryReporter.java
@@ -0,0 +1,159 @@
+package org.apache.blur.utils;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Counter;
+import com.yammer.metrics.core.Gauge;
+import com.yammer.metrics.core.Histogram;
+import com.yammer.metrics.core.Metered;
+import com.yammer.metrics.core.Metric;
+import com.yammer.metrics.core.MetricName;
+import com.yammer.metrics.core.MetricProcessor;
+import com.yammer.metrics.core.MetricsRegistry;
+import com.yammer.metrics.core.Timer;
+import com.yammer.metrics.reporting.AbstractPollingReporter;
+import com.yammer.metrics.stats.Snapshot;
+
+public class MemoryReporter extends AbstractPollingReporter implements
+    MetricProcessor<ConcurrentMap<String, org.apache.blur.thrift.generated.Metric>>
{
+
+  private static final Log LOG = LogFactory.getLog(MemoryReporter.class);
+
+  private static ConcurrentMap<String, org.apache.blur.thrift.generated.Metric> _metrics
= new ConcurrentHashMap<String, org.apache.blur.thrift.generated.Metric>();
+
+  public static Map<String, org.apache.blur.thrift.generated.Metric> getMetrics() {
+    return new HashMap<String, org.apache.blur.thrift.generated.Metric>(_metrics);
+  }
+
+  public static void enable() {
+    MemoryReporter memoryReporter = new MemoryReporter(Metrics.defaultRegistry(), "memory-reporter");
+    memoryReporter.start(1, TimeUnit.SECONDS);
+  }
+
+  protected MemoryReporter(MetricsRegistry registry, String name) {
+    super(registry, name);
+  }
+
+  @Override
+  public void run() {
+    try {
+      MetricsRegistry registry = getMetricsRegistry();
+      Map<MetricName, Metric> allMetrics = registry.allMetrics();
+      for (Entry<MetricName, Metric> entry : allMetrics.entrySet()) {
+        entry.getValue().processWith(this, entry.getKey(), _metrics);
+      }
+    } catch (Exception e) {
+      LOG.error("Unknown error during metrics processing.", e);
+    }
+  }
+
+  @Override
+  public void processMeter(MetricName name, Metered meter,
+      ConcurrentMap<String, org.apache.blur.thrift.generated.Metric> context) throws
Exception {
+    org.apache.blur.thrift.generated.Metric metric = getMetric(name, context);
+    addMeter(metric, meter, context);
+  }
+
+  private org.apache.blur.thrift.generated.Metric getMetric(MetricName name,
+      ConcurrentMap<String, org.apache.blur.thrift.generated.Metric> context) {
+    String nameStr = name.toString();
+    org.apache.blur.thrift.generated.Metric metric = context.get(nameStr);
+    if (metric == null) {
+      metric = new org.apache.blur.thrift.generated.Metric();
+      context.put(nameStr, metric);
+      metric.setName(nameStr);
+    }
+    return metric;
+  }
+
+  private void addMeter(org.apache.blur.thrift.generated.Metric metric, Metered meter,
+      ConcurrentMap<String, org.apache.blur.thrift.generated.Metric> context) {
+    metric.putToStrMap("rateUnit", meter.rateUnit().toString());
+    metric.putToStrMap("eventType", meter.eventType());
+    metric.putToLongMap("count", meter.count());
+    metric.putToDoubleMap("meanRate", meter.meanRate());
+    metric.putToDoubleMap("oneMinuteRate", meter.oneMinuteRate());
+    metric.putToDoubleMap("fiveMinuteRate", meter.fiveMinuteRate());
+    metric.putToDoubleMap("fifteenMinuteRate", meter.fifteenMinuteRate());
+  }
+
+  @Override
+  public void processCounter(MetricName name, Counter counter,
+      ConcurrentMap<String, org.apache.blur.thrift.generated.Metric> context) throws
Exception {
+    org.apache.blur.thrift.generated.Metric metric = getMetric(name, context);
+    metric.putToLongMap("value", counter.count());
+  }
+
+  @Override
+  public void processHistogram(MetricName name, Histogram histogram,
+      ConcurrentMap<String, org.apache.blur.thrift.generated.Metric> context) throws
Exception {
+    org.apache.blur.thrift.generated.Metric metric = getMetric(name, context);
+    metric.putToDoubleMap("min", histogram.min());
+    metric.putToDoubleMap("max", histogram.max());
+    metric.putToDoubleMap("mean", histogram.mean());
+    metric.putToDoubleMap("stdDev", histogram.stdDev());
+
+    Snapshot snapshot = histogram.getSnapshot();
+    metric.putToDoubleMap("median", snapshot.getMedian());
+    metric.putToDoubleMap("75%", snapshot.get75thPercentile());
+    metric.putToDoubleMap("95%", snapshot.get95thPercentile());
+    metric.putToDoubleMap("98%", snapshot.get98thPercentile());
+    metric.putToDoubleMap("99%", snapshot.get99thPercentile());
+    metric.putToDoubleMap("99.9%", snapshot.get999thPercentile());
+  }
+
+  @Override
+  public void processTimer(MetricName name, Timer timer,
+      ConcurrentMap<String, org.apache.blur.thrift.generated.Metric> context) throws
Exception {
+
+    org.apache.blur.thrift.generated.Metric metric = getMetric(name, context);
+    addMeter(metric, timer, context);
+    metric.putToStrMap("unit", timer.durationUnit().toString());
+    metric.putToDoubleMap("min", timer.min());
+    metric.putToDoubleMap("max", timer.max());
+    metric.putToDoubleMap("mean", timer.mean());
+    metric.putToDoubleMap("stdDev", timer.stdDev());
+
+    Snapshot snapshot = timer.getSnapshot();
+    metric.putToDoubleMap("median", snapshot.getMedian());
+    metric.putToDoubleMap("75%", snapshot.get75thPercentile());
+    metric.putToDoubleMap("95%", snapshot.get95thPercentile());
+    metric.putToDoubleMap("98%", snapshot.get98thPercentile());
+    metric.putToDoubleMap("99%", snapshot.get99thPercentile());
+    metric.putToDoubleMap("99.9%", snapshot.get999thPercentile());
+  }
+
+  @Override
+  public void processGauge(MetricName name, Gauge<?> gauge,
+      ConcurrentMap<String, org.apache.blur.thrift.generated.Metric> context) throws
Exception {
+    org.apache.blur.thrift.generated.Metric metric = getMetric(name, context);
+    metric.putToDoubleMap("value", getDouble(gauge.value()));
+  }
+
+  private double getDouble(Object value) {
+    if (value instanceof Integer) {
+      Integer v = (Integer) value;
+      return (int) v;
+    } else if (value instanceof Long) {
+      Long v = (Long) value;
+      return (long) v;
+    } else if (value instanceof Double) {
+      Double v = (Double) value;
+      return v;
+    } else if (value instanceof Float) {
+      Float v = (Float) value;
+      return (float) v;
+    }
+    return 0;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-shell/src/main/java/org/apache/blur/shell/Main.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/Main.java b/blur-shell/src/main/java/org/apache/blur/shell/Main.java
index bd4ed23..0a6958d 100644
--- a/blur-shell/src/main/java/org/apache/blur/shell/Main.java
+++ b/blur-shell/src/main/java/org/apache/blur/shell/Main.java
@@ -49,6 +49,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableMap.Builder;
 
 public class Main {
+  static final String PROMPT = "blur> ";
   /** is debugging enabled - off by default */
   static boolean debug = false;
   /** is timing enabled - off by default */
@@ -213,7 +214,7 @@ public class Main {
       int bufferLength = getMaxCommandLength(cmds.keySet()) + 2;
       out.println(" - Table commands - ");
       String[] tableCommands = { "create", "enable", "disable", "remove", "truncate", "describe",
"list", "schema",
-          "stats", "layout" };
+          "stats", "layout", "parse" };
       printCommandAndHelp(out, cmds, tableCommands, bufferLength);
 
       out.println();
@@ -223,7 +224,7 @@ public class Main {
 
       out.println();
       out.println(" - Cluster commands - ");
-      String[] clusterCommands = { "controllers", "shards", "clusterlist", "cluster", "safemodewait"
};
+      String[] clusterCommands = { "controllers", "shards", "clusterlist", "cluster", "safemodewait",
"top" };
       printCommandAndHelp(out, cmds, clusterCommands, bufferLength);
 
       out.println();
@@ -239,10 +240,11 @@ public class Main {
           out.println("  " + buffer(e.getKey(), bufferLength) + " - " + e.getValue().help());
         }
       }
-      
+
       out.println();
       out.println("  " + buffer("shell", bufferLength) + " - enters into the Blur interactive
shell");
-      out.println("  " + buffer("execute", bufferLength) + " - executes a custom class passing
all the command line args to the main method");
+      out.println("  " + buffer("execute", bufferLength)
+          + " - executes a custom class passing all the command line args to the main method");
     }
 
     private int getMaxCommandLength(Set<String> keySet) {
@@ -326,6 +328,8 @@ public class Main {
     builder.put("truncate", new TruncateTableCommand());
     builder.put("cluster", new ClusterCommand());
     builder.put("safemodewait", new WaitInSafemodeCommand());
+    builder.put("top", new TopCommand());
+    builder.put("parse", new ParseCommand());
     commands = builder.build();
 
     CliShellOptions cliShellOptions = getCliShellOptions(args);
@@ -338,7 +342,7 @@ public class Main {
       if (cliShellOptions.isShell()) {
         ConsoleReader reader = new ConsoleReader();
         setConsoleReader(commands, reader);
-        reader.setPrompt("blur> ");
+        reader.setPrompt(PROMPT);
 
         List<Completer> completors = new LinkedList<Completer>();
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-shell/src/main/java/org/apache/blur/shell/ParseCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/ParseCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/ParseCommand.java
new file mode 100644
index 0000000..e651a81
--- /dev/null
+++ b/blur-shell/src/main/java/org/apache/blur/shell/ParseCommand.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.blur.shell;
+
+import java.io.PrintWriter;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.Blur;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.SimpleQuery;
+
+public class ParseCommand extends Command {
+  @Override
+  public void doit(PrintWriter out, Blur.Iface client, String[] args) throws CommandException,
TException,
+      BlurException {
+    if (args.length < 3) {
+      throw new CommandException("Invalid args: " + help());
+    }
+    String tablename = args[1];
+    String queryStr = "";
+    for (int i = 2; i < args.length; i++) {
+      queryStr += args[i] + " ";
+    }
+
+    SimpleQuery simpleQuery = new SimpleQuery();
+    simpleQuery.setQueryStr(queryStr);
+    String query = client.parseQuery(tablename, simpleQuery);
+    out.println(query);
+  }
+
+  @Override
+  public String help() {
+    return "parse a query and return string representation, args; tablename query";
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7137006b/blur-shell/src/main/java/org/apache/blur/shell/TopCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/TopCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/TopCommand.java
new file mode 100644
index 0000000..a5108f8
--- /dev/null
+++ b/blur-shell/src/main/java/org/apache/blur/shell/TopCommand.java
@@ -0,0 +1,277 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.blur.shell;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import jline.Terminal;
+import jline.console.ConsoleReader;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.BlurClient;
+import org.apache.blur.thrift.generated.Blur;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.Metric;
+
+public class TopCommand extends Command {
+
+  public enum SCREEN {
+    HELP, TOP
+  }
+
+  private static final String SC = "seg cnt";
+  private static final String IC = "idx cnt";
+  private static final String TC = "tble cnt";
+  private static final String RE = "rd rec";
+  private static final String IM = "idx mem";
+  private static final String RO = "rd row";
+  private static final String CS = "bc size";
+  private static final String CE = "bc evt";
+  private static final String CM = "bc miss";
+  private static final String CH = "bc hit";
+  private static final String IQ = "in qry";
+  private static final String EQ = "ex qry";
+  private static final String SHARD_SERVER = "Shard Server";
+  private static final String INTERNAL_QUERIES = "\"org.apache.blur\":type=\"Blur\",name=\"Internal
Queries/s\"";
+  private static final String EXTERNAL_QUERIES = "\"org.apache.blur\":type=\"Blur\",name=\"External
Queries/s\"";
+
+  private static final String CACHE_HIT = "\"org.apache.blur\":type=\"Cache\",name=\"Hit\"";
+  private static final String CACHE_MISS = "\"org.apache.blur\":type=\"Cache\",name=\"Miss\"";
+  private static final String CACHE_EVICTION = "\"org.apache.blur\":type=\"Cache\",name=\"Eviction\"";
+  private static final String CACHE_SIZE = "\"org.apache.blur\":type=\"Cache\",name=\"Size\"";
+
+  private static final String READ_RECORDS = "\"org.apache.blur\":type=\"Blur\",name=\"Read
Records/s\"";
+  private static final String READ_ROWS = "\"org.apache.blur\":type=\"Blur\",name=\"Read
Row/s\"";
+
+  private static final String INDEX_MEMORY_USAGE = "\"org.apache.blur\":type=\"Blur\",scope=\"default\",name=\"Index
Memory Usage\"";
+  private static final String TABLE_COUNT = "\"org.apache.blur\":type=\"Blur\",scope=\"default\",name=\"Table
Count\"";
+  private static final String INDEX_COUNT = "\"org.apache.blur\":type=\"Blur\",scope=\"default\",name=\"Index
Count\"";
+  private static final String SEGMENT_COUNT = "\"org.apache.blur\":type=\"Blur\",scope=\"default\",name=\"Segment
Count\"";
+  private static final double ONE_MILLION = 1000000;
+  private static final double ONE_BILLION = 1000 * ONE_MILLION;
+  private static final double ONE_TRILLION = 1000 * ONE_BILLION;
+  private static final double ONE_QUADRILLION = 1000 * ONE_TRILLION;
+
+  private int _width = Integer.MAX_VALUE;
+  private int _height;
+
+  @Override
+  public void doit(PrintWriter out, Blur.Iface client, String[] args) throws CommandException,
TException,
+      BlurException {
+    try {
+      doitInternal(out, client, args);
+    } finally {
+      ConsoleReader reader = this.getConsoleReader();
+      if (reader != null) {
+        reader.setPrompt(Main.PROMPT);
+      }
+    }
+  }
+
+  public void doitInternal(PrintWriter out, Blur.Iface client, String[] args) throws CommandException,
TException,
+      BlurException {
+
+    AtomicBoolean quit = new AtomicBoolean();
+
+    Map<String, String> metricNames = new HashMap<String, String>();
+    metricNames.put(IQ, INTERNAL_QUERIES);
+    metricNames.put(EQ, EXTERNAL_QUERIES);
+    metricNames.put(CH, CACHE_HIT);
+    metricNames.put(CM, CACHE_MISS);
+    metricNames.put(CE, CACHE_EVICTION);
+    metricNames.put(CS, CACHE_SIZE);
+    metricNames.put(RO, READ_RECORDS);
+    metricNames.put(RE, READ_ROWS);
+    metricNames.put(IM, INDEX_MEMORY_USAGE);
+    metricNames.put(TC, TABLE_COUNT);
+    metricNames.put(IC, INDEX_COUNT);
+    metricNames.put(SC, SEGMENT_COUNT);
+
+    Object[] labels = new Object[] { SHARD_SERVER, EQ, IQ, CH, CM, CE, CS, RO, RE, IM, TC,
IC, SC };
+
+    Set<String> sizes = new HashSet<String>();
+    sizes.add(IM);
+
+    Set<String> keys = new HashSet<String>(metricNames.values());
+
+    String cluster;
+    if (args.length != 2) {
+      cluster = Main.getCluster(client, "Invalid args: " + help());
+    } else {
+      cluster = args[1];
+    }
+
+    List<String> shardServerList = client.shardServerList(cluster);
+
+    ConsoleReader reader = this.getConsoleReader();
+    if (reader != null) {
+      Terminal terminal = reader.getTerminal();
+      _height = terminal.getHeight() - 2;
+      _width = terminal.getWidth() - 2;
+      try {
+        reader.setPrompt("");
+        reader.clearScreen();
+      } catch (IOException e) {
+        if (Main.debug) {
+          e.printStackTrace();
+        }
+      }
+      startCommandWatcher(reader, quit, this);
+    }
+
+    Map<String, Blur.Iface> shardClients = new HashMap<String, Blur.Iface>();
+    for (String sc : shardServerList) {
+      shardClients.put(sc, BlurClient.getClient(sc));
+    }
+
+    int longestServerName = Math.max(getSizeOfLongestKey(shardClients), SHARD_SERVER.length());
+
+    StringBuilder header = new StringBuilder("%" + longestServerName + "s");
+    for (int i = 1; i < labels.length; i++) {
+      header.append(" %10s");
+    }
+    header.append("%n");
+
+    do {
+      if (quit.get()) {
+        return;
+      }
+      out.printf(truncate(header.toString()), labels);
+      for (Entry<String, Blur.Iface> e : shardClients.entrySet()) {
+        String shardServer = e.getKey();
+        Iface shardClient = e.getValue();
+        Object[] cols = new Object[labels.length];
+        int c = 0;
+        cols[c++] = shardServer;
+        StringBuilder sb = new StringBuilder("%" + longestServerName + "s");
+
+        Map<String, Metric> metrics = shardClient.metrics(keys);
+        for (int i = 1; i < labels.length; i++) {
+          String mn = metricNames.get(labels[i]);
+          Metric metric = metrics.get(mn);
+          Map<String, Double> doubleMap = metric.getDoubleMap();
+          Double value = doubleMap.get("oneMinuteRate");
+          if (value == null) {
+            value = doubleMap.get("value");
+          }
+          cols[c++] = humanize(value, sizes.contains(mn));
+          sb.append(" %10s");
+        }
+        sb.append("%n");
+        out.printf(truncate(sb.toString()), cols);
+      }
+      out.flush();
+      if (reader != null) {
+        try {
+          synchronized (this) {
+            wait(3000);
+          }
+        } catch (InterruptedException e) {
+          return;
+        }
+        try {
+          reader.clearScreen();
+        } catch (IOException e) {
+          if (Main.debug) {
+            e.printStackTrace();
+          }
+        }
+        Terminal terminal = reader.getTerminal();
+        _height = terminal.getHeight() - 2;
+        _width = terminal.getWidth() - 2;
+      }
+    } while (reader != null);
+
+  }
+
+  private void startCommandWatcher(final ConsoleReader reader, final AtomicBoolean quit,
final Object lock) {
+    Thread thread = new Thread(new Runnable() {
+      @Override
+      public void run() {
+        try {
+          while (true) {
+            int readCharacter = reader.readCharacter();
+            if (readCharacter == 'q') {
+              quit.set(true);
+              synchronized (lock) {
+                lock.notify();
+              }
+              return;
+            }
+          }
+        } catch (IOException e) {
+          if (Main.debug) {
+            e.printStackTrace();
+          }
+        }
+      }
+    });
+    thread.setDaemon(true);
+    thread.start();
+  }
+
+  private String truncate(String s) {
+    return s.substring(0, Math.min(_width, s.length()));
+  }
+
+  private String humanize(double value, boolean size) {
+    long v = (long) (value / ONE_QUADRILLION);
+    if (v > 0) {
+      return String.format("%7.2f%s", value / ONE_QUADRILLION, size ? "Q" : "P");
+    }
+    v = (long) (value / ONE_TRILLION);
+    if (v > 0) {
+      return String.format("%7.2f%s", value / ONE_TRILLION, "T");
+    }
+    v = (long) (value / ONE_BILLION);
+    if (v > 0) {
+      return String.format("%7.2f%s", value / ONE_BILLION, size ? "B" : "G");
+    }
+    v = (long) (value / ONE_MILLION);
+    if (v > 0) {
+      return String.format("%7.2f%s", value / ONE_MILLION, "M");
+    }
+    return String.format("%7.2f", value);
+  }
+
+  private int getSizeOfLongestKey(Map<String, ?> map) {
+    int i = 0;
+    for (String s : map.keySet()) {
+      int length = s.length();
+      if (i < length) {
+        i = length;
+      }
+    }
+    return i;
+  }
+
+  @Override
+  public String help() {
+    return "top arg; cluster";
+  }
+}


Mime
View raw message