kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject [3/4] kudu git commit: metrics: fast-path PB/JSON dump for empty histograms
Date Sat, 03 Feb 2018 01:09:29 GMT
metrics: fast-path PB/JSON dump for empty histograms

It turns out that a decent percentage of our histograms are empty in
real life use cases. For example, some tablets might be cold enough that
they have never had any action since the tserver started. Other
histograms might correspond to features which aren't being used in a
particular workload (eg commit-wait counters).

For example, in a test cluster running a cycling YCSB workload, about
20% of histograms are empty. In another test cluster running more of an
analytic workload with a lot of very cold tablets, 81% of histograms
were empty.

We can fast-path various calculations on these histograms to just return
0 for a simple speedup on the /metricsjson endpoint.

Change-Id: I0732e8299e2f27c7c6021ec70740984ab46a69c0
Reviewed-on: http://gerrit.cloudera.org:8080/9179
Tested-by: Kudu Jenkins
Reviewed-by: Will Berkeley <wdberkeley@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/afae5c75
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/afae5c75
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/afae5c75

Branch: refs/heads/master
Commit: afae5c75112c9938ed9f2b335a75a92ea63f5587
Parents: 3e49fe2
Author: Todd Lipcon <todd@apache.org>
Authored: Wed Jan 31 18:02:03 2018 -0800
Committer: Todd Lipcon <todd@apache.org>
Committed: Sat Feb 3 01:08:32 2018 +0000

----------------------------------------------------------------------
 src/kudu/util/metrics.cc | 58 +++++++++++++++++++++++++++----------------
 src/kudu/util/metrics.h  |  2 +-
 2 files changed, 38 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/afae5c75/src/kudu/util/metrics.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/metrics.cc b/src/kudu/util/metrics.cc
index 4954bac..6565d97 100644
--- a/src/kudu/util/metrics.cc
+++ b/src/kudu/util/metrics.cc
@@ -627,34 +627,50 @@ Status Histogram::WriteAsJson(JsonWriter* writer,
 
 Status Histogram::GetHistogramSnapshotPB(HistogramSnapshotPB* snapshot_pb,
                                          const MetricJsonOptions& opts) const {
-  HdrHistogram snapshot(*histogram_);
   snapshot_pb->set_name(prototype_->name());
   if (opts.include_schema_info) {
     snapshot_pb->set_type(MetricType::Name(prototype_->type()));
     snapshot_pb->set_label(prototype_->label());
     snapshot_pb->set_unit(MetricUnit::Name(prototype_->unit()));
     snapshot_pb->set_description(prototype_->description());
-    snapshot_pb->set_max_trackable_value(snapshot.highest_trackable_value());
-    snapshot_pb->set_num_significant_digits(snapshot.num_significant_digits());
+    snapshot_pb->set_max_trackable_value(histogram_->highest_trackable_value());
+    snapshot_pb->set_num_significant_digits(histogram_->num_significant_digits());
   }
-  snapshot_pb->set_total_count(snapshot.TotalCount());
-  snapshot_pb->set_total_sum(snapshot.TotalSum());
-  snapshot_pb->set_min(snapshot.MinValue());
-  snapshot_pb->set_mean(snapshot.MeanValue());
-  snapshot_pb->set_percentile_75(snapshot.ValueAtPercentile(75));
-  snapshot_pb->set_percentile_95(snapshot.ValueAtPercentile(95));
-  snapshot_pb->set_percentile_99(snapshot.ValueAtPercentile(99));
-  snapshot_pb->set_percentile_99_9(snapshot.ValueAtPercentile(99.9));
-  snapshot_pb->set_percentile_99_99(snapshot.ValueAtPercentile(99.99));
-  snapshot_pb->set_max(snapshot.MaxValue());
-
-  if (opts.include_raw_histograms) {
-    RecordedValuesIterator iter(&snapshot);
-    while (iter.HasNext()) {
-      HistogramIterationValue value;
-      RETURN_NOT_OK(iter.Next(&value));
-      snapshot_pb->add_values(value.value_iterated_to);
-      snapshot_pb->add_counts(value.count_at_value_iterated_to);
+  // Fast-path for a reasonably common case of an empty histogram. This occurs
+  // when a histogram is tracking some information about a feature not in
+  // use, for example.
+  if (histogram_->TotalCount() == 0) {
+    snapshot_pb->set_total_count(0);
+    snapshot_pb->set_total_sum(0);
+    snapshot_pb->set_min(0);
+    snapshot_pb->set_mean(0);
+    snapshot_pb->set_percentile_75(0);
+    snapshot_pb->set_percentile_95(0);
+    snapshot_pb->set_percentile_99(0);
+    snapshot_pb->set_percentile_99_9(0);
+    snapshot_pb->set_percentile_99_99(0);
+    snapshot_pb->set_max(0);
+  } else {
+    HdrHistogram snapshot(*histogram_);
+    snapshot_pb->set_total_count(snapshot.TotalCount());
+    snapshot_pb->set_total_sum(snapshot.TotalSum());
+    snapshot_pb->set_min(snapshot.MinValue());
+    snapshot_pb->set_mean(snapshot.MeanValue());
+    snapshot_pb->set_percentile_75(snapshot.ValueAtPercentile(75));
+    snapshot_pb->set_percentile_95(snapshot.ValueAtPercentile(95));
+    snapshot_pb->set_percentile_99(snapshot.ValueAtPercentile(99));
+    snapshot_pb->set_percentile_99_9(snapshot.ValueAtPercentile(99.9));
+    snapshot_pb->set_percentile_99_99(snapshot.ValueAtPercentile(99.99));
+    snapshot_pb->set_max(snapshot.MaxValue());
+
+    if (opts.include_raw_histograms) {
+      RecordedValuesIterator iter(&snapshot);
+      while (iter.HasNext()) {
+        HistogramIterationValue value;
+        RETURN_NOT_OK(iter.Next(&value));
+        snapshot_pb->add_values(value.value_iterated_to);
+        snapshot_pb->add_counts(value.count_at_value_iterated_to);
+      }
     }
   }
   return Status::OK();

http://git-wip-us.apache.org/repos/asf/kudu/blob/afae5c75/src/kudu/util/metrics.h
----------------------------------------------------------------------
diff --git a/src/kudu/util/metrics.h b/src/kudu/util/metrics.h
index 60412c2..aa553c9 100644
--- a/src/kudu/util/metrics.h
+++ b/src/kudu/util/metrics.h
@@ -1009,7 +1009,7 @@ class Histogram : public Metric {
                              const MetricJsonOptions& opts) const OVERRIDE;
 
   // Returns a snapshot of this histogram including the bucketed values and counts.
-  Status GetHistogramSnapshotPB(HistogramSnapshotPB* snapshot,
+  Status GetHistogramSnapshotPB(HistogramSnapshotPB* snapshot_pb,
                                 const MetricJsonOptions& opts) const;
 
   uint64_t CountInBucketForValueForTests(uint64_t value) const;


Mime
View raw message