chukwa-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject [11/22] chukwa git commit: CHUKWA-758. Updated default dashboard to include system metrics. (Eric Yang)
Date Thu, 25 Jun 2015 20:48:42 GMT
CHUKWA-758.  Updated default dashboard to include system metrics.  (Eric Yang)


Project: http://git-wip-us.apache.org/repos/asf/chukwa/repo
Commit: http://git-wip-us.apache.org/repos/asf/chukwa/commit/d8abe979
Tree: http://git-wip-us.apache.org/repos/asf/chukwa/tree/d8abe979
Diff: http://git-wip-us.apache.org/repos/asf/chukwa/diff/d8abe979

Branch: refs/heads/master
Commit: d8abe979aaa2f107240ea5aeaeca9207eff65332
Parents: 0cad3aa
Author: Eric Yang <eyang@apache.org>
Authored: Thu Jun 18 20:38:30 2015 -0700
Committer: Eric Yang <eyang@apache.org>
Committed: Thu Jun 18 20:38:30 2015 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |   2 +
 .../writer/hbase/HBaseWriter.java               |   2 +-
 .../chukwa/datastore/ChukwaHBaseStore.java      | 138 ++++++++++++++++---
 .../hbase/HadoopMetricsProcessor.java           |  74 +++++-----
 src/main/web/hicc/home/css/component.css        |   1 +
 src/main/web/hicc/home/index.html               |   2 +-
 6 files changed, 161 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/chukwa/blob/d8abe979/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ef7af42..adcd624 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -42,6 +42,8 @@ Trunk (unreleased changes)
 
   BUGS
 
+    CHUKWA-758.  Updated default dashboard to include system metrics.  (Eric Yang)
+
     CHUKWA-743. Revised pid locking using standard Hadoop approach.  (Eric Yang)
 
     CHUKWA-742. Updated license header for StatusCheckerException.java and exclude 

http://git-wip-us.apache.org/repos/asf/chukwa/blob/d8abe979/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
index e5a8083..0718feb 100644
--- a/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
+++ b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
@@ -98,7 +98,7 @@ public class HBaseWriter extends PipelineableWriter {
     this.statTimer = new Timer();
     this.defaultProcessor = conf.get(
       "chukwa.demux.mapper.default.processor",
-      "org.apache.hadoop.chukwa.extraction.demux.processor.mapper.DefaultProcessor");
+      "org.apache.hadoop.chukwa.extraction.hbase.DefaultProcessor");
     log.info("hbase.zookeeper.quorum: " + hconf.get(HConstants.ZOOKEEPER_QUORUM) + ":" +
hconf.get(HConstants.ZOOKEEPER_CLIENT_PORT));
     if (reportStats) {
       statTimer.schedule(new StatReportingTask(), 1000, 10 * 1000);

http://git-wip-us.apache.org/repos/asf/chukwa/blob/d8abe979/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java b/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java
index 66553c4..d75a350 100644
--- a/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java
+++ b/src/main/java/org/apache/hadoop/chukwa/datastore/ChukwaHBaseStore.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.chukwa.datastore;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Calendar;
@@ -240,7 +241,7 @@ public class ChukwaHBaseStore {
         Result result = it.next();
         for (Cell cell : result.rawCells()) {
           JSONObject json = (JSONObject) JSONValue.parse(new String(cell.getValue(), "UTF-8"));
-          if (json.get("type").equals("source")) {
+          if (json!=null && json.get("type")!=null && json.get("type").equals("source"))
{
             pk.add(new String(cell.getQualifier(), "UTF-8"));
           }
         }
@@ -418,6 +419,30 @@ public class ChukwaHBaseStore {
   }
 
   /**
+   * Create a chart in HBase by specifying parameters.
+   * @throws URISyntaxException 
+   */
+  public static synchronized String createChart(String id, String yunitType, 
+      String title, String[] metrics, String source) throws URISyntaxException {
+    Chart chart = new Chart(id);
+    chart.setYUnitType(yunitType);
+    chart.setTitle(title);
+    ArrayList<SeriesMetaData> series = new ArrayList<SeriesMetaData>();
+    for(String metric : metrics) {
+      SeriesMetaData s = new SeriesMetaData();
+      s.setLabel(metric + "/" + source);
+      s.setUrl(new URI("/hicc/v1/metrics/series/" + metric + "/"
+        + source));
+      LineOptions l = new LineOptions();
+      s.setLineOptions(l);
+      series.add(s);
+    }
+    chart.SetSeries(series);
+    return createChart(chart);
+    
+  }
+
+  /**
    * Create a chart in HBase.
    * 
    * @param chart
@@ -719,13 +744,27 @@ public class ChukwaHBaseStore {
   public static void populateDefaults() {
     try {
       String hostname = InetAddress.getLocalHost().getHostName();
+      // Populate example chart widgets
+      String[] metrics = { "SystemMetrics.LoadAverage.1" };
+      createChart("1", "", "System Load Average", metrics, hostname);
+      String[] cpuMetrics = { "SystemMetrics.cpu.combined", "SystemMetrics.cpu.sys", "SystemMetrics.cpu.user"
};
+      createChart("2", "%", "CPU Utilization", cpuMetrics, hostname);
+      String[] memMetrics = { "SystemMetrics.memory.FreePercent", "SystemMetrics.memory.UsedPercent"};
+      createChart("3", "%", "Memory Utilization", memMetrics, hostname);
+      String[] diskMetrics = { "SystemMetrics.disk.ReadBytes", "SystemMetrics.disk.WriteBytes"
};
+      createChart("4", "bytes-decimal", "Disk Utilization", diskMetrics, hostname);
+      String[] netMetrics = { "SystemMetrics.network.TxBytes", "SystemMetrics.network.RxBytes"
};
+      createChart("5", "bytes", "Network Utilization", netMetrics, hostname);
+      String[] swapMetrics = { "SystemMetrics.swap.Total", "SystemMetrics.swap.Used", "SystemMetrics.swap.Free"
};
+      createChart("6", "", "Swap Utilization", swapMetrics, hostname);
+      
       // Populate default widgets
       Widget widget = new Widget();
       widget.setTitle("System Load Average");
       widget.setSrc(new URI("/hicc/v1/chart/draw/1"));
       widget.setCol(1);
       widget.setRow(1);
-      widget.setSize_x(7);
+      widget.setSize_x(2);
       widget.setSize_y(1);
       createWidget(widget);
 
@@ -733,32 +772,89 @@ public class ChukwaHBaseStore {
       widget = new Widget();
       widget.setTitle("CPU Heatmap");
       widget.setSrc(new URI("/hicc/jsp/heatmap.jsp"));
+      widget.setCol(3);
+      widget.setRow(1);
+      widget.setSize_x(5);
+      widget.setSize_y(1);
+      createWidget(widget);
+
+      Dashboard dashboard = new Dashboard();
+      // Log Search widget
+      widget = new Widget();
+      widget.setTitle("Log Search");
+      widget.setSrc(new URI("/hicc/ajax-solr/chukwa"));
+      widget.setCol(1);
+      widget.setRow(4);
+      widget.setSize_x(7);
+      widget.setSize_y(4);
+      createWidget(widget);
+      dashboard.add(widget);
+      updateDashboard("default", "", dashboard);
+
+      // Populate system dashboards
+      dashboard = new Dashboard();
+      widget = new Widget();
+      widget.setTitle("CPU Utilization");
+      widget.setSrc(new URI("/hicc/v1/chart/draw/2"));
       widget.setCol(1);
       widget.setRow(1);
-      widget.setSize_x(4);
-      widget.setSize_y(2);
+      widget.setSize_x(3);
+      widget.setSize_y(1);
       createWidget(widget);
+      dashboard.add(widget);
 
-      // Populate example chart widgets
-      Chart chart = new Chart("1");
-      chart.setYUnitType("");
-      chart.setTitle("Load Average");
-      ArrayList<SeriesMetaData> series = new ArrayList<SeriesMetaData>();
+      widget = new Widget();
+      widget.setTitle("Memory Utilization");
+      widget.setSrc(new URI("/hicc/v1/chart/draw/3"));
+      widget.setCol(4);
+      widget.setRow(1);
+      widget.setSize_x(3);
+      widget.setSize_y(1);
+      createWidget(widget);
+      dashboard.add(widget);
 
-      SeriesMetaData s = new SeriesMetaData();
-      s.setLabel("SystemMetrics.LoadAverage.1/" + hostname);
-      s.setUrl(new URI("/hicc/v1/metrics/series/SystemMetrics.LoadAverage.1/"
-          + hostname));
-      LineOptions l = new LineOptions();
-      s.setLineOptions(l);
-      series.add(s);
+      widget = new Widget();
+      widget.setTitle("Disk Utilization");
+      widget.setSrc(new URI("/hicc/v1/chart/draw/4"));
+      widget.setCol(1);
+      widget.setRow(2);
+      widget.setSize_x(3);
+      widget.setSize_y(1);
+      createWidget(widget);
+      dashboard.add(widget);
 
-      chart.SetSeries(series);
-      createChart(chart);
-      
-      Dashboard dashboard = new Dashboard();
+      widget = new Widget();
+      widget.setTitle("Network Utilization");
+      widget.setSrc(new URI("/hicc/v1/chart/draw/5"));
+      widget.setCol(4);
+      widget.setRow(2);
+      widget.setSize_x(3);
+      widget.setSize_y(1);
+      createWidget(widget);
       dashboard.add(widget);
-      updateDashboard("default", "", dashboard);
+
+      widget = new Widget();
+      widget.setTitle("Swap Utilization");
+      widget.setSrc(new URI("/hicc/v1/chart/draw/6"));
+      widget.setCol(1);
+      widget.setRow(3);
+      widget.setSize_x(3);
+      widget.setSize_y(1);
+      createWidget(widget);
+      dashboard.add(widget);
+
+      widget = new Widget();
+      widget.setTitle("System Load Average");
+      widget.setSrc(new URI("/hicc/v1/chart/draw/1"));
+      widget.setCol(4);
+      widget.setRow(3);
+      widget.setSize_x(3);
+      widget.setSize_y(1);
+      createWidget(widget);
+      dashboard.add(widget);
+
+      updateDashboard("system", "", dashboard);
+      
     } catch (Throwable ex) {
       LOG.error(ExceptionUtil.getStackTrace(ex));
     }

http://git-wip-us.apache.org/repos/asf/chukwa/blob/d8abe979/src/main/java/org/apache/hadoop/chukwa/extraction/hbase/HadoopMetricsProcessor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/chukwa/extraction/hbase/HadoopMetricsProcessor.java
b/src/main/java/org/apache/hadoop/chukwa/extraction/hbase/HadoopMetricsProcessor.java
index 3afd71a..19df607 100644
--- a/src/main/java/org/apache/hadoop/chukwa/extraction/hbase/HadoopMetricsProcessor.java
+++ b/src/main/java/org/apache/hadoop/chukwa/extraction/hbase/HadoopMetricsProcessor.java
@@ -35,6 +35,8 @@ public class HadoopMetricsProcessor extends AbstractProcessor {
   static final String timestampField = "timestamp";
   static final String contextNameField = "contextName";
   static final String recordNameField = "recordName";
+  static final String hostName = "Hostname";
+  static final String processName = "ProcessName";
   static final byte[] cf = "t".getBytes();
 
   public HadoopMetricsProcessor() throws NoSuchAlgorithmException {
@@ -43,44 +45,46 @@ public class HadoopMetricsProcessor extends AbstractProcessor {
   @Override
   protected void parse(byte[] recordEntry) throws Throwable {
     try {
-    	String body = new String(recordEntry);
-        int start = body.indexOf('{');
-        JSONObject json = (JSONObject) JSONValue.parse(body.substring(start));
+      String body = new String(recordEntry);
+      int start = body.indexOf('{');
+      JSONObject json = (JSONObject) JSONValue.parse(body.substring(start));
 
-        time = ((Long) json.get(timestampField)).longValue();
-        String contextName = (String) json.get(contextNameField);
-        String recordName = (String) json.get(recordNameField);
-        byte[] timeInBytes = ByteBuffer.allocate(8).putLong(time).array();
-
-        @SuppressWarnings("unchecked")
-		Iterator<String> ki = json.keySet().iterator();
-        while (ki.hasNext()) {
-          String keyName = ki.next();
-          if (timestampField.intern() == keyName.intern()) {
-        	  continue;
-          } else if (contextNameField.intern() == keyName.intern()) {
-        	  continue;
-          } else if (recordNameField.intern() == keyName.intern()) {
-        	  continue;
-          } else {
-            if(json.get(keyName)!=null) {
-                byte[] v = json.get(keyName).toString().getBytes();
-                String primaryKey = new StringBuilder(contextName).append(".").
-              		  append(recordName).append(".").
-              		  append(keyName).toString();
-                byte[] rowKey = HBaseUtil.buildKey(time, primaryKey, chunk.getSource());
-                Put r = new Put(rowKey);
-                r.add(cf, timeInBytes, time, v);
-                output.add(r);
-            }
+      time = ((Long) json.get(timestampField)).longValue();
+      String contextName = (String) json.get(contextNameField);
+      String recordName = (String) json.get(recordNameField);
+      String src = (String) json.get(hostName);
+      if(json.get(processName)!=null) {
+        src = new StringBuilder(src).append(":").append(json.get(processName)).toString();
+      }
+      @SuppressWarnings("unchecked")
+      Iterator<String> ki = json.keySet().iterator();
+      while (ki.hasNext()) {
+        String keyName = ki.next();
+        if (timestampField.intern() == keyName.intern()) {
+          continue;
+        } else if (contextNameField.intern() == keyName.intern()) {
+          continue;
+        } else if (recordNameField.intern() == keyName.intern()) {
+          continue;
+        } else if (hostName.intern() == keyName.intern()) {
+          continue;
+        } else if (processName.intern() == keyName.intern()) {
+          continue;
+        } else {
+          if (json.get(keyName) != null) {
+            String v = json.get(keyName).toString();
+            String primaryKey = new StringBuilder(contextName).append(".")
+                .append(recordName).append(".").append(keyName).toString();
+            addRecord(time, primaryKey, src, v.getBytes(), output);
           }
         }
-        
-      } catch (Exception e) {
-        LOG.warn("Wrong format in HadoopMetricsProcessor [" + recordEntry + "]",
-            e);
-        throw e;
-      }	
+      }
+
+    } catch (Exception e) {
+      LOG.warn("Wrong format in HadoopMetricsProcessor [" + recordEntry + "]",
+          e);
+      throw e;
+    }
   }
 
 }

http://git-wip-us.apache.org/repos/asf/chukwa/blob/d8abe979/src/main/web/hicc/home/css/component.css
----------------------------------------------------------------------
diff --git a/src/main/web/hicc/home/css/component.css b/src/main/web/hicc/home/css/component.css
index 98c38bc..9828873 100755
--- a/src/main/web/hicc/home/css/component.css
+++ b/src/main/web/hicc/home/css/component.css
@@ -602,6 +602,7 @@ input.gn-search:focus {
     left: 0;
     z-index: 99;
     background: transparent;
+    display: none;
 }
 
 .frame {

http://git-wip-us.apache.org/repos/asf/chukwa/blob/d8abe979/src/main/web/hicc/home/index.html
----------------------------------------------------------------------
diff --git a/src/main/web/hicc/home/index.html b/src/main/web/hicc/home/index.html
index 22a6e52..78c117c 100755
--- a/src/main/web/hicc/home/index.html
+++ b/src/main/web/hicc/home/index.html
@@ -33,7 +33,7 @@
     <script src="js/jquery.gridster.js" type="text/javascript" charset="utf-8"></script>
     <script src="js/typeahead.bundle.js" type="text/javascript"></script>
   </head>
-  <body>
+  <body onunload="">
     <div id="glass" class="glass" onclick="toggleGlass()"></div>
     <div class="frame">
       <ul><li><iframe id="frame"></iframe></li></ul>


Mime
View raw message