impala-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From k...@apache.org
Subject [1/4] incubator-impala git commit: IMPALA-4884: Add JVM heap and non-heap usage in metrics and UI
Date Mon, 13 Feb 2017 19:23:24 GMT
Repository: incubator-impala
Updated Branches:
  refs/heads/master 6242478d0 -> f982c3f76


IMPALA-4884: Add JVM heap and non-heap usage in metrics and UI

This commit adds heap and non-heap memory usage of the embedded
JVM in the memory metrics and exposes these metrics in /memz web page of
the impalad and catalog web UI.

Change-Id: I543d4d428d7240e0f710d67973867162f2fcabc8
Reviewed-on: http://gerrit.cloudera.org:8080/5909
Reviewed-by: Dimitris Tsirogiannis <dtsirogiannis@cloudera.com>
Tested-by: Impala Public Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/af469984
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/af469984
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/af469984

Branch: refs/heads/master
Commit: af4699847090ff7d03a588aa1b2e48170c64f03f
Parents: 6242478
Author: Dimitris Tsirogiannis <dtsirogiannis@cloudera.com>
Authored: Sat Feb 4 21:14:05 2017 -0800
Committer: Impala Public Jenkins <impala-public-jenkins@gerrit.cloudera.org>
Committed: Sat Feb 11 01:34:13 2017 +0000

----------------------------------------------------------------------
 be/src/util/default-path-handlers.cc            | 12 +++-
 .../java/org/apache/impala/common/JniUtil.java  | 36 ++++++++++
 www/memz.tmpl                                   | 70 ++++++++++++++++++--
 3 files changed, 110 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/af469984/be/src/util/default-path-handlers.cc
----------------------------------------------------------------------
diff --git a/be/src/util/default-path-handlers.cc b/be/src/util/default-path-handlers.cc
index 91c51d3..732d3b3 100644
--- a/be/src/util/default-path-handlers.cc
+++ b/be/src/util/default-path-handlers.cc
@@ -148,17 +148,23 @@ void MemUsageHandler(MemTracker* mem_tracker, MetricGroup* metric_group,
     if (jvm_group != NULL) {
       Value jvm(kObjectType);
       jvm_group->ToJson(false, document, &jvm);
+      Value heap(kArrayType);
+      Value non_heap(kArrayType);
       Value total(kArrayType);
       for (SizeType i = 0; i < jvm["metrics"].Size(); ++i) {
         if (strstr(jvm["metrics"][i]["name"].GetString(), "total") != nullptr) {
           total.PushBack(jvm["metrics"][i], document->GetAllocator());
+        } else if (strstr(jvm["metrics"][i]["name"].GetString(), "non-heap") != nullptr)
{
+          non_heap.PushBack(jvm["metrics"][i], document->GetAllocator());
+        } else if (strstr(jvm["metrics"][i]["name"].GetString(), "heap") != nullptr) {
+          heap.PushBack(jvm["metrics"][i], document->GetAllocator());
         }
       }
-      document->AddMember("jvm", total, document->GetAllocator());
-
+      document->AddMember("jvm_total", total, document->GetAllocator());
+      document->AddMember("jvm_heap", heap, document->GetAllocator());
+      document->AddMember("jvm_non_heap", non_heap, document->GetAllocator());
     }
   }
-
 }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/af469984/fe/src/main/java/org/apache/impala/common/JniUtil.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/common/JniUtil.java b/fe/src/main/java/org/apache/impala/common/JniUtil.java
index 4446a55..223c8bb 100644
--- a/fe/src/main/java/org/apache/impala/common/JniUtil.java
+++ b/fe/src/main/java/org/apache/impala/common/JniUtil.java
@@ -22,6 +22,7 @@ import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.io.Writer;
 import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
 import java.lang.management.MemoryPoolMXBean;
 import java.lang.management.MemoryUsage;
 import java.util.ArrayList;
@@ -142,6 +143,41 @@ public class JniUtil {
         }
       }
     }
+
+    if (request.get_all || request.getMemory_pool().equals("heap")) {
+      // Populate heap usage
+      MemoryMXBean mBean = ManagementFactory.getMemoryMXBean();
+      TJvmMemoryPool heap = new TJvmMemoryPool();
+      MemoryUsage heapUsage = mBean.getHeapMemoryUsage();
+      heap.setCommitted(heapUsage.getCommitted());
+      heap.setInit(heapUsage.getInit());
+      heap.setMax(heapUsage.getMax());
+      heap.setUsed(heapUsage.getUsed());
+      heap.setName("heap");
+      heap.setPeak_committed(0);
+      heap.setPeak_init(0);
+      heap.setPeak_max(0);
+      heap.setPeak_used(0);
+      jvmMetrics.getMemory_pools().add(heap);
+    }
+
+    if (request.get_all || request.getMemory_pool().equals("non-heap")) {
+      // Populate non-heap usage
+      MemoryMXBean mBean = ManagementFactory.getMemoryMXBean();
+      TJvmMemoryPool nonHeap = new TJvmMemoryPool();
+      MemoryUsage nonHeapUsage = mBean.getNonHeapMemoryUsage();
+      nonHeap.setCommitted(nonHeapUsage.getCommitted());
+      nonHeap.setInit(nonHeapUsage.getInit());
+      nonHeap.setMax(nonHeapUsage.getMax());
+      nonHeap.setUsed(nonHeapUsage.getUsed());
+      nonHeap.setName("non-heap");
+      nonHeap.setPeak_committed(0);
+      nonHeap.setPeak_init(0);
+      nonHeap.setPeak_max(0);
+      nonHeap.setPeak_used(0);
+      jvmMetrics.getMemory_pools().add(nonHeap);
+    }
+
     TSerializer serializer = new TSerializer(protocolFactory_);
     try {
       return serializer.serialize(jvmMetrics);

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/af469984/www/memz.tmpl
----------------------------------------------------------------------
diff --git a/www/memz.tmpl b/www/memz.tmpl
index f54cdaa..48bd651 100644
--- a/www/memz.tmpl
+++ b/www/memz.tmpl
@@ -28,15 +28,15 @@ Memory consumption / limit: <strong>{{consumption}}</strong>
/ <strong>{{mem_lim
 <h3>Breakdown</h3>
 <pre>{{detailed}}</pre>
 
-{{?jvm}}
-<h3>JVM memory total usage</h3>
+{{?jvm_heap}}
+<h3>JVM heap usage</h3>
 <table class='table table-bordered table-hover'>
   <tr>
     <th>Name</th>
     <th>Value</th>
     <th>Description</th>
   </tr>
-  {{#jvm}}
+  {{#jvm_heap}}
   <tr>
     <td><tt>{{name}}</tt></td>
     {{! Is this a stats metric? }}
@@ -54,8 +54,68 @@ Memory consumption / limit: <strong>{{consumption}}</strong>
/ <strong>{{mem_lim
       {{description}}
     </td>
   </tr>
-  {{/jvm}}
+  {{/jvm_heap}}
 </table>
-{{/jvm}}
+{{/jvm_heap}}
+
+{{?jvm_non_heap}}
+<h3>JVM non-heap usage</h3>
+<table class='table table-bordered table-hover'>
+  <tr>
+    <th>Name</th>
+    <th>Value</th>
+    <th>Description</th>
+  </tr>
+  {{#jvm_non_heap}}
+  <tr>
+    <td><tt>{{name}}</tt></td>
+    {{! Is this a stats metric? }}
+    {{?mean}}
+    <td>
+      Last (of {{count}}): <strong>{{last}}</strong>.
+      Min: {{min}}, max: {{max}}, avg: {{mean}}</td>
+    {{/mean}}
+    {{^mean}}
+    <td>
+      {{human_readable}}
+    </td>
+    {{/mean}}
+    <td>
+      {{description}}
+    </td>
+  </tr>
+  {{/jvm_non_heap}}
+</table>
+{{/jvm_non_heap}}
+
+{{?jvm_total}}
+<h3>JVM total memory usage</h3>
+<table class='table table-bordered table-hover'>
+  <tr>
+    <th>Name</th>
+    <th>Value</th>
+    <th>Description</th>
+  </tr>
+  {{#jvm_total}}
+  <tr>
+    <td><tt>{{name}}</tt></td>
+    {{! Is this a stats metric? }}
+    {{?mean}}
+    <td>
+      Last (of {{count}}): <strong>{{last}}</strong>.
+      Min: {{min}}, max: {{max}}, avg: {{mean}}</td>
+    {{/mean}}
+    {{^mean}}
+    <td>
+      {{human_readable}}
+    </td>
+    {{/mean}}
+    <td>
+      {{description}}
+    </td>
+  </tr>
+  {{/jvm_total}}
+</table>
+{{/jvm_total}}
 
 {{> www/common-footer.tmpl }}


Mime
View raw message