incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [4/7] git commit: Changes needed for live updates.
Date Tue, 11 Sep 2012 02:59:57 GMT
Changes needed for live updates.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/b0eda0de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/b0eda0de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/b0eda0de

Branch: refs/heads/master
Commit: b0eda0de39cd0726d96749d53d287f3087aa2785
Parents: dcee9d6
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Mon Sep 10 22:38:05 2012 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Mon Sep 10 22:38:05 2012 -0400

----------------------------------------------------------------------
 .../java/org/apache/blur/manager/IndexManager.java |   24 +
 .../apache/blur/thrift/ThriftBlurShardServer.java  |   37 +-
 .../java/org/apache/blur/gui/HttpJettyServer.java  |    2 +-
 .../org/apache/blur/gui/LiveMetricsServlet.java    |   61 +
 src/blur-gui/src/main/webapp/d3.v2.js              |16001 ++++++---------
 src/blur-gui/src/main/webapp/home.jsp              |    6 +-
 src/blur-gui/src/main/webapp/livemetrics.jsp       |  111 +
 src/blur-gui/src/main/webapp/metrics.jsp           |    6 +-
 src/blur-gui/src/main/webapp/shardList.jsp         |    6 +-
 src/blur-gui/src/main/webapp/table.jsp             |    6 +-
 .../java/org/apache/blur/metrics/HeapMetrics.java  |   81 +
 .../java/org/apache/blur/metrics/QueryMetrics.java |  142 +
 .../org/apache/blur/metrics/SystemLoadMetrics.java |   72 +
 13 files changed, 7339 insertions(+), 9216 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0eda0de/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
index 2bbb646..0470db5 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
@@ -57,6 +57,7 @@ import org.apache.blur.manager.status.QueryStatus;
 import org.apache.blur.manager.status.QueryStatusManager;
 import org.apache.blur.manager.writer.BlurIndex;
 import org.apache.blur.metrics.BlurMetrics;
+import org.apache.blur.metrics.QueryMetrics;
 import org.apache.blur.thrift.BException;
 import org.apache.blur.thrift.MutationHelper;
 import org.apache.blur.thrift.generated.BlurException;
@@ -120,7 +121,9 @@ public class IndexManager {
   private BlurPartitioner<BytesWritable, Void> _blurPartitioner = new BlurPartitioner<BytesWritable,
Void>();
   private BlurFilterCache _filterCache = new DefaultBlurFilterCache();
   private BlurMetrics _blurMetrics;
+  private QueryMetrics _queryMetrics;
   private long _defaultParallelCallTimeout = TimeUnit.MINUTES.toMillis(1);
+  
 
   public void setMaxClauseCount(int maxClauseCount) {
     BooleanQuery.setMaxClauseCount(maxClauseCount);
@@ -131,6 +134,7 @@ public class IndexManager {
     // @TODO give the mutate it's own thread pool
     _mutateExecutor = Executors.newThreadPool("index-manager-mutate", _threadCount);
     _statusManager.init();
+    _queryMetrics = QueryMetrics.getInstance();
     LOG.info("Init Complete");
   }
 
@@ -179,7 +183,9 @@ public class IndexManager {
     IndexReader reader = null;
     try {
       reader = index.getIndexReader();
+      long s = System.nanoTime();
       fetchRow(reader, table, selector, fetchResult);
+      _queryMetrics.recordDataFetch(System.nanoTime() - s, getRecordCount(fetchResult));
       if (_blurMetrics != null) {
         if (fetchResult.rowResult != null) {
           if (fetchResult.rowResult.row != null && fetchResult.rowResult.row.records
!= null) {
@@ -205,6 +211,17 @@ public class IndexManager {
     }
   }
 
+  private long getRecordCount(FetchResult fetchResult) {
+    if (fetchResult.rowResult != null) {
+      if (fetchResult.rowResult.row != null && fetchResult.rowResult.row.records
!= null) {
+        return fetchResult.rowResult.row.records.size();
+      }
+    } else if (fetchResult.recordResult != null) {
+      return 1;
+    }
+    return 0;
+  }
+
   private void populateSelector(String table, Selector selector) throws IOException, BlurException
{
     String rowId = selector.rowId;
     String recordId = selector.recordId;
@@ -286,6 +303,7 @@ public class IndexManager {
   }
 
   public BlurResultIterable query(final String table, final BlurQuery blurQuery, AtomicLongArray
facetedCounts) throws Exception {
+    long s = System.nanoTime();
     final AtomicBoolean running = new AtomicBoolean(true);
     final QueryStatus status = _statusManager.newQueryStatus(table, blurQuery, _threadCount,
running);
     _blurMetrics.queriesExternal.incrementAndGet();
@@ -327,6 +345,7 @@ public class IndexManager {
       }).merge(merger);
     } finally {
       _statusManager.removeStatus(status);
+      _queryMetrics.recordQuery(System.nanoTime() - s);
     }
   }
 
@@ -730,16 +749,19 @@ public class IndexManager {
         waitVisiblity = waitToBeVisible;
       }
       RowMutationType type = mutation.rowMutationType;
+      long start = System.nanoTime();
       switch (type) {
       case REPLACE_ROW:
         Row row = MutationHelper.getRowFromMutations(mutation.rowId, mutation.recordMutations);
         blurIndex.replaceRow(waitVisiblity, mutation.wal, row);
+        _queryMetrics.recordDataMutate(System.nanoTime() - start, row.records.size());
         break;
       case UPDATE_ROW:
         doUpdateRowMutation(mutation, blurIndex);
         break;
       case DELETE_ROW:
         blurIndex.deleteRow(waitVisiblity, mutation.wal, mutation.rowId);
+        _queryMetrics.recordDataMutate(System.nanoTime() - start, 1);
         break;
       default:
         throw new RuntimeException("Not supported [" + type + "]");
@@ -842,8 +864,10 @@ public class IndexManager {
         }
       }
 
+      long s = System.nanoTime();
       // Finally, replace the existing row with the new row we have built.
       blurIndex.replaceRow(mutation.waitToBeVisible, mutation.wal, newRow);
+      _queryMetrics.recordDataMutate(System.nanoTime() - s, newRow.records.size());
     } else {
       throw new BException("Mutation cannot update row that does not exist.", mutation);
     }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0eda0de/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
index d92c394..7a6ea44 100644
--- a/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/thrift/ThriftBlurShardServer.java
@@ -101,7 +101,28 @@ public class ThriftBlurShardServer extends ThriftServer {
     int slabCount = configuration.getInt(BLUR_SHARD_BLOCKCACHE_SLAB_COUNT, 1);
     Cache cache;
     Configuration config = new Configuration();
+    
+    String bindAddress = configuration.get(BLUR_SHARD_BIND_ADDRESS);
+    int bindPort = configuration.getInt(BLUR_SHARD_BIND_PORT, -1);
+    bindPort += serverIndex;
+    
     BlurMetrics blurMetrics = new BlurMetrics(config);
+    
+    int baseGuiPort = Integer.parseInt(configuration.get(BLUR_GUI_SHARD_PORT));
+    final HttpJettyServer httpServer;
+    if (baseGuiPort > 0) {
+      int webServerPort = baseGuiPort + serverIndex;
+
+      // TODO: this got ugly, there has to be a better way to handle all these
+      // params
+      // without reversing the mvn dependancy and making blur-gui on top.
+      httpServer = new HttpJettyServer(bindPort, webServerPort, configuration.getInt(BLUR_CONTROLLER_BIND_PORT,
-1), configuration.getInt(BLUR_SHARD_BIND_PORT, -1),
+          configuration.getInt(BLUR_GUI_CONTROLLER_PORT, -1), configuration.getInt(BLUR_GUI_SHARD_PORT,
-1), "shard", blurMetrics);
+    } else {
+      httpServer = null;
+    }
+    
+    
     if (slabCount >= 1) {
       BlockCache blockCache;
       boolean directAllocation = configuration.getBoolean(BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION,
true);
@@ -128,9 +149,6 @@ public class ThriftBlurShardServer extends ThriftServer {
       cache = BlockDirectory.NO_CACHE;
     }
 
-    String bindAddress = configuration.get(BLUR_SHARD_BIND_ADDRESS);
-    int bindPort = configuration.getInt(BLUR_SHARD_BIND_PORT, -1);
-    bindPort += serverIndex;
 
     LOG.info("Shard Server using index [{0}] bind address [{1}]", serverIndex, bindAddress
+ ":" + bindPort);
 
@@ -209,19 +227,6 @@ public class ThriftBlurShardServer extends ThriftServer {
     server.setIface(iface);
     server.setConfiguration(configuration);
 
-    int baseGuiPort = Integer.parseInt(configuration.get(BLUR_GUI_SHARD_PORT));
-    final HttpJettyServer httpServer;
-    if (baseGuiPort > 0) {
-      int webServerPort = baseGuiPort + serverIndex;
-
-      // TODO: this got ugly, there has to be a better way to handle all these
-      // params
-      // without reversing the mvn dependancy and making blur-gui on top.
-      httpServer = new HttpJettyServer(bindPort, webServerPort, configuration.getInt(BLUR_CONTROLLER_BIND_PORT,
-1), configuration.getInt(BLUR_SHARD_BIND_PORT, -1),
-          configuration.getInt(BLUR_GUI_CONTROLLER_PORT, -1), configuration.getInt(BLUR_GUI_SHARD_PORT,
-1), "shard", blurMetrics);
-    } else {
-      httpServer = null;
-    }
 
     // This will shutdown the server when the correct path is set in zk
     BlurShutdown shutdown = new BlurShutdown() {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0eda0de/src/blur-gui/src/main/java/org/apache/blur/gui/HttpJettyServer.java
----------------------------------------------------------------------
diff --git a/src/blur-gui/src/main/java/org/apache/blur/gui/HttpJettyServer.java b/src/blur-gui/src/main/java/org/apache/blur/gui/HttpJettyServer.java
index 37b51ef..d775bb8 100644
--- a/src/blur-gui/src/main/java/org/apache/blur/gui/HttpJettyServer.java
+++ b/src/blur-gui/src/main/java/org/apache/blur/gui/HttpJettyServer.java
@@ -27,7 +27,6 @@ import org.mortbay.jetty.Server;
 import org.mortbay.jetty.servlet.ServletHolder;
 import org.mortbay.jetty.webapp.WebAppContext;
 
-
 /**
  * Starts up a Jetty server to run the utility gui.
  * 
@@ -76,6 +75,7 @@ public class HttpJettyServer {
     context.setWar(warPath);
     context.setContextPath("/");
     context.setParentLoaderPriority(true);
+    context.addServlet(new ServletHolder(new LiveMetricsServlet()), "/livemetrics");
     context.addServlet(new ServletHolder(new MetricsServlet(bm)), "/metrics");
     context.addServlet(new ServletHolder(new LogServlet(blurLogFile)), "/logs");
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0eda0de/src/blur-gui/src/main/java/org/apache/blur/gui/LiveMetricsServlet.java
----------------------------------------------------------------------
diff --git a/src/blur-gui/src/main/java/org/apache/blur/gui/LiveMetricsServlet.java b/src/blur-gui/src/main/java/org/apache/blur/gui/LiveMetricsServlet.java
new file mode 100644
index 0000000..0a234f6
--- /dev/null
+++ b/src/blur-gui/src/main/java/org/apache/blur/gui/LiveMetricsServlet.java
@@ -0,0 +1,61 @@
+package org.apache.blur.gui;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.blur.metrics.HeapMetrics;
+import org.apache.blur.metrics.QueryMetrics;
+import org.apache.blur.metrics.SystemLoadMetrics;
+
+public class LiveMetricsServlet extends HttpServlet {
+
+  private static final long serialVersionUID = 1L;
+
+  private HeapMetrics heapMetrics;
+  private SystemLoadMetrics systemLoadMetrics;
+  private QueryMetrics queryMetrics;
+
+  public LiveMetricsServlet() {
+    heapMetrics = HeapMetrics.getInstance();
+    systemLoadMetrics = SystemLoadMetrics.getInstance();
+    queryMetrics = QueryMetrics.getInstance();
+  }
+
+  protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException,
IOException {
+    response.setContentType("application/json");
+    PrintWriter out = response.getWriter();
+    out.write("{");
+    out.write("\"jvm\":{\"xLabel\":\"Time\",\"yLabel\":\"Heap (GB)\",\"lines\":");
+    heapMetrics.writeJson(out);
+    out.write("}");
+    out.write(",\"blur\":{\"xLabel\":\"Time\",\"yLabel\":\"Rates\",\"lines\":");
+    queryMetrics.writeJson(out);
+    out.write("}");
+    out.write(",\"system\":{\"xLabel\":\"Time\",\"yLabel\":\"Load\",\"lines\":");
+    systemLoadMetrics.writeJson(out);
+    out.write("}");
+    out.write("}");
+  }
+
+}


Mime
View raw message