Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 3EDBD200C09 for ; Wed, 25 Jan 2017 20:49:00 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 3D5FD160B4E; Wed, 25 Jan 2017 19:49:00 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id EE54B160B3D for ; Wed, 25 Jan 2017 20:48:57 +0100 (CET) Received: (qmail 12349 invoked by uid 500); 25 Jan 2017 19:48:43 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 12310 invoked by uid 99); 25 Jan 2017 19:48:43 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 25 Jan 2017 19:48:43 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 348EAE080A; Wed, 25 Jan 2017 19:48:43 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: enis@apache.org To: commits@hbase.apache.org Date: Wed, 25 Jan 2017 19:48:45 -0000 Message-Id: In-Reply-To: <68d75411177d443eb74c9c0bcdc8c352@git.apache.org> References: <68d75411177d443eb74c9c0bcdc8c352@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [3/3] hbase git commit: HBASE-9774 HBase native metrics and metric collection for coprocessors archived-at: Wed, 25 Jan 2017 19:49:00 -0000 HBASE-9774 HBase native metrics and metric collection for coprocessors Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c64a1d19 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c64a1d19 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c64a1d19 Branch: refs/heads/master Commit: c64a1d199402d6bf2d6ff4168c00c756dcaa59e4 Parents: 59fd6eb Author: Enis Soztutar Authored: Wed Jan 25 11:47:35 2017 -0800 Committer: Enis Soztutar Committed: Wed Jan 25 11:47:35 2017 -0800 ---------------------------------------------------------------------- hbase-assembly/pom.xml | 10 + .../hadoop/hbase/util/FastLongHistogram.java | 310 ----------- .../hbase/util/TestFastLongHistogram.java | 132 ----- .../ExampleMasterObserverWithMetrics.java | 142 +++++ .../ExampleRegionObserverWithMetrics.java | 121 +++++ hbase-hadoop-compat/pom.xml | 4 + .../apache/hadoop/hbase/metrics/BaseSource.java | 5 + hbase-hadoop2-compat/pom.xml | 4 + .../hadoop/hbase/metrics/BaseSourceImpl.java | 46 ++ .../impl/GlobalMetricRegistriesAdapter.java | 172 ++++++ .../impl/HBaseMetrics2HadoopMetricsAdapter.java | 169 ++++++ .../MetricsRegionServerSourceImpl.java | 6 + .../lib/DefaultMetricsSystemHelper.java | 50 +- .../hadoop/metrics2/lib/MutableHistogram.java | 110 ++-- .../metrics2/lib/MutableRangeHistogram.java | 14 +- hbase-metrics-api/README.txt | 78 +++ hbase-metrics-api/pom.xml | 112 ++++ .../apache/hadoop/hbase/metrics/Counter.java | 60 ++ .../org/apache/hadoop/hbase/metrics/Gauge.java | 35 ++ .../apache/hadoop/hbase/metrics/Histogram.java | 58 ++ .../org/apache/hadoop/hbase/metrics/Meter.java | 90 +++ .../org/apache/hadoop/hbase/metrics/Metric.java | 30 + .../hadoop/hbase/metrics/MetricRegistries.java | 89 +++ .../hbase/metrics/MetricRegistriesLoader.java | 96 ++++ .../hadoop/hbase/metrics/MetricRegistry.java | 111 ++++ .../hbase/metrics/MetricRegistryFactory.java | 36 ++ .../hbase/metrics/MetricRegistryInfo.java | 112 ++++ .../apache/hadoop/hbase/metrics/MetricSet.java | 41 ++ .../hadoop/hbase/metrics/PackageMarker.java | 36 ++ .../apache/hadoop/hbase/metrics/Snapshot.java | 136 +++++ .../org/apache/hadoop/hbase/metrics/Timer.java | 68 +++ .../hadoop/hbase/metrics/package-info.java | 25 + .../metrics/TestMetricRegistriesLoader.java | 56 ++ hbase-metrics/README.txt | 1 + hbase-metrics/pom.xml | 136 +++++ .../hadoop/hbase/metrics/impl/CounterImpl.java | 61 +++ .../hbase/metrics/impl/DropwizardMeter.java | 74 +++ .../hbase/metrics/impl/FastLongHistogram.java | 397 ++++++++++++++ .../hbase/metrics/impl/HistogramImpl.java | 81 +++ .../metrics/impl/MetricRegistriesImpl.java | 75 +++ .../metrics/impl/MetricRegistryFactoryImpl.java | 34 ++ .../hbase/metrics/impl/MetricRegistryImpl.java | 122 +++++ .../hbase/metrics/impl/RefCountingMap.java | 91 ++++ .../hadoop/hbase/metrics/impl/TimerImpl.java | 58 ++ .../hadoop/hbase/metrics/impl/package-info.java | 25 + ...apache.hadoop.hbase.metrics.MetricRegistries | 18 + .../hbase/metrics/impl/TestCounterImpl.java | 59 ++ .../hbase/metrics/impl/TestDropwizardMeter.java | 51 ++ .../metrics/impl/TestFastLongHistogram.java | 132 +++++ .../hadoop/hbase/metrics/impl/TestGauge.java | 61 +++ .../hbase/metrics/impl/TestHistogramImpl.java | 100 ++++ .../metrics/impl/TestMetricRegistryImpl.java | 142 +++++ .../hbase/metrics/impl/TestRefCountingMap.java | 154 ++++++ .../hbase/metrics/impl/TestTimerImpl.java | 53 ++ hbase-server/pom.xml | 8 + .../MasterCoprocessorEnvironment.java | 10 + .../hbase/coprocessor/MetricsCoprocessor.java | 136 +++++ .../RegionCoprocessorEnvironment.java | 24 +- .../RegionServerCoprocessorEnvironment.java | 10 + .../coprocessor/WALCoprocessorEnvironment.java | 10 + .../hadoop/hbase/io/hfile/AgeSnapshot.java | 2 +- .../hadoop/hbase/io/hfile/BlockCacheUtil.java | 2 +- .../hadoop/hbase/io/hfile/CacheStats.java | 2 +- .../hbase/master/MasterCoprocessorHost.java | 27 +- .../hbase/regionserver/MetricsRegionServer.java | 21 +- .../hbase/regionserver/RSRpcServices.java | 8 + .../regionserver/RegionCoprocessorHost.java | 21 +- .../RegionServerCoprocessorHost.java | 26 +- .../regionserver/wal/WALCoprocessorHost.java | 16 + .../apache/hadoop/hbase/MiniHBaseCluster.java | 7 + .../coprocessor/TestCoprocessorMetrics.java | 544 +++++++++++++++++++ .../security/token/TestTokenAuthentication.java | 6 + pom.xml | 26 + 73 files changed, 4848 insertions(+), 547 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml index b9d8dcc..19bf5f0 100644 --- a/hbase-assembly/pom.xml +++ b/hbase-assembly/pom.xml @@ -208,6 +208,16 @@ org.apache.hbase + hbase-metrics-api + ${project.version} + + + org.apache.hbase + hbase-metrics + ${project.version} + + + org.apache.hbase hbase-resource-bundle ${project.version} true http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java deleted file mode 100644 index 310348e..0000000 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java +++ /dev/null @@ -1,310 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.util; - -import java.util.Arrays; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.LongAdder; -import java.util.stream.Stream; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; - -/** - * FastLongHistogram is a thread-safe class that estimate distribution of data and computes the - * quantiles. - */ -@InterfaceAudience.Public -@InterfaceStability.Evolving -public class FastLongHistogram { - - /** - * Default number of bins. - */ - public static final int DEFAULT_NBINS = 255; - - public static final double[] DEFAULT_QUANTILES = - new double[]{0.25, 0.5, 0.75, 0.90, 0.95, 0.98, 0.99, 0.999}; - - /** - * Bins is a class containing a list of buckets(or bins) for estimation histogram of some data. - */ - private static class Bins { - private final LongAdder[] counts; - // inclusive - private final long binsMin; - // exclusive - private final long binsMax; - private final long bins10XMax; - private final AtomicLong min = new AtomicLong(Long.MAX_VALUE); - private final AtomicLong max = new AtomicLong(0L); - - private final LongAdder count = new LongAdder(); - private final LongAdder total = new LongAdder(); - - // set to true when any of data has been inserted to the Bins. It is set after the counts are - // updated. - private volatile boolean hasData = false; - - /** - * The constructor for creating a Bins without any prior data. - */ - public Bins(int numBins) { - counts = createCounters(numBins); - this.binsMin = 1L; - - // These two numbers are total guesses - // and should be treated as highly suspect. - this.binsMax = 1000; - this.bins10XMax = binsMax * 10; - } - - /** - * The constructor for creating a Bins with last Bins. - */ - public Bins(Bins last, int numBins, double minQ, double maxQ) { - long[] values = last.getQuantiles(new double[] { minQ, maxQ }); - long wd = values[1] - values[0] + 1; - // expand minQ and maxQ in two ends back assuming uniform distribution - this.binsMin = Math.max(0L, (long) (values[0] - wd * minQ)); - long binsMax = (long) (values[1] + wd * (1 - maxQ)) + 1; - // make sure each of bins is at least of width 1 - this.binsMax = Math.max(binsMax, this.binsMin + numBins); - this.bins10XMax = Math.max((long) (values[1] + (binsMax - 1) * 9), this.binsMax + 1); - - this.counts = createCounters(numBins); - } - - private LongAdder[] createCounters(int numBins) { - return Stream.generate(LongAdder::new).limit(numBins + 3).toArray(LongAdder[]::new); - } - - private int getIndex(long value) { - if (value < this.binsMin) { - return 0; - } else if (value > this.bins10XMax) { - return this.counts.length - 1; - } else if (value >= this.binsMax) { - return this.counts.length - 2; - } - // compute the position - return 1 + (int) ((value - this.binsMin) * (this.counts.length - 3) / - (this.binsMax - this.binsMin)); - - } - - /** - * Adds a value to the histogram. - */ - public void add(long value, long count) { - if (value < 0) { - // The whole computation is completely thrown off if there are negative numbers - // - // Normally we would throw an IllegalArgumentException however this is the metrics - // system and it should be completely safe at all times. - // So silently throw it away. - return; - } - AtomicUtils.updateMin(min, value); - AtomicUtils.updateMax(max, value); - - this.count.add(count); - this.total.add(value * count); - - int pos = getIndex(value); - this.counts[pos].add(count); - - // hasData needs to be updated as last - this.hasData = true; - } - - /** - * Computes the quantiles give the ratios. - */ - public long[] getQuantiles(double[] quantiles) { - if (!hasData) { - // No data yet. - return new long[quantiles.length]; - } - - // Make a snapshot of lowerCounter, higherCounter and bins.counts to counts. - // This is not synchronized, but since the counter are accumulating, the result is a good - // estimation of a snapshot. - long[] counts = new long[this.counts.length]; - long total = 0L; - for (int i = 0; i < this.counts.length; i++) { - counts[i] = this.counts[i].sum(); - total += counts[i]; - } - - int rIndex = 0; - double qCount = total * quantiles[0]; - long cum = 0L; - - long[] res = new long[quantiles.length]; - countsLoop: for (int i = 0; i < counts.length; i++) { - // mn and mx define a value range - long mn, mx; - if (i == 0) { - mn = this.min.get(); - mx = this.binsMin; - } else if (i == counts.length - 1) { - mn = this.bins10XMax; - mx = this.max.get(); - } else if (i == counts.length - 2) { - mn = this.binsMax; - mx = this.bins10XMax; - } else { - mn = this.binsMin + (i - 1) * (this.binsMax - this.binsMin) / (this.counts.length - 3); - mx = this.binsMin + i * (this.binsMax - this.binsMin) / (this.counts.length - 3); - } - - if (mx < this.min.get()) { - continue; - } - if (mn > this.max.get()) { - break; - } - mn = Math.max(mn, this.min.get()); - mx = Math.min(mx, this.max.get()); - - // lastCum/cum are the corresponding counts to mn/mx - double lastCum = cum; - cum += counts[i]; - - // fill the results for qCount is within current range. - while (qCount <= cum) { - if (cum == lastCum) { - res[rIndex] = mn; - } else { - res[rIndex] = (long) ((qCount - lastCum) * (mx - mn) / (cum - lastCum) + mn); - } - - // move to next quantile - rIndex++; - if (rIndex >= quantiles.length) { - break countsLoop; - } - qCount = total * quantiles[rIndex]; - } - } - // In case quantiles contains values >= 100% - for (; rIndex < quantiles.length; rIndex++) { - res[rIndex] = this.max.get(); - } - - return res; - } - - long getNumAtOrBelow(long val) { - return Arrays.stream(counts).mapToLong(c -> c.sum()).limit(getIndex(val) + 1).sum(); - } - } - - // The bins counting values. It is replaced with a new one in calling of reset(). - private volatile Bins bins; - - /** - * Constructor. - */ - public FastLongHistogram() { - this(DEFAULT_NBINS); - } - - /** - * Constructor. - * @param numOfBins the number of bins for the histogram. A larger value results in more precise - * results but with lower efficiency, and vice versus. - */ - public FastLongHistogram(int numOfBins) { - this.bins = new Bins(numOfBins); - } - - /** - * Constructor setting the bins assuming a uniform distribution within a range. - * @param numOfBins the number of bins for the histogram. A larger value results in more precise - * results but with lower efficiency, and vice versus. - * @param min lower bound of the region, inclusive. - * @param max higher bound of the region, inclusive. - */ - public FastLongHistogram(int numOfBins, long min, long max) { - this(numOfBins); - Bins bins = new Bins(numOfBins); - bins.add(min, 1); - bins.add(max, 1); - this.bins = new Bins(bins, numOfBins, 0.01, 0.999); - } - - private FastLongHistogram(Bins bins) { - this.bins = bins; - } - - /** - * Adds a value to the histogram. - */ - public void add(long value, long count) { - this.bins.add(value, count); - } - - /** - * Computes the quantiles give the ratios. - */ - public long[] getQuantiles(double[] quantiles) { - return this.bins.getQuantiles(quantiles); - } - - public long[] getQuantiles() { - return this.bins.getQuantiles(DEFAULT_QUANTILES); - } - - public long getMin() { - long min = this.bins.min.get(); - return min == Long.MAX_VALUE ? 0 : min; // in case it is not initialized - } - - public long getMax() { - return this.bins.max.get(); - } - - public long getCount() { - return this.bins.count.sum(); - } - - public long getMean() { - Bins bins = this.bins; - long count = bins.count.sum(); - long total = bins.total.sum(); - if (count == 0) { - return 0; - } - return total / count; - } - - public long getNumAtOrBelow(long value) { - return this.bins.getNumAtOrBelow(value); - } - - /** - * Resets the histogram for new counting. - */ - public FastLongHistogram reset() { - Bins oldBins = this.bins; - this.bins = new Bins(this.bins, this.bins.counts.length - 3, 0.01, 0.99); - return new FastLongHistogram(oldBins); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java deleted file mode 100644 index d56d143..0000000 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.util; - -import java.util.Arrays; -import java.util.Random; - -import org.apache.hadoop.hbase.testclassification.MiscTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import static org.junit.Assert.assertEquals; - -/** - * Testcases for FastLongHistogram. - */ -@Category({MiscTests.class, SmallTests.class}) -public class TestFastLongHistogram { - - private static void doTestUniform(FastLongHistogram hist) { - long[] VALUES = { 0, 10, 20, 30, 40, 50 }; - double[] qs = new double[VALUES.length]; - for (int i = 0; i < qs.length; i++) { - qs[i] = (double) VALUES[i] / VALUES[VALUES.length - 1]; - } - - for (int i = 0; i < 10; i++) { - for (long v : VALUES) { - hist.add(v, 1); - } - long[] vals = hist.getQuantiles(qs); - System.out.println(Arrays.toString(vals)); - for (int j = 0; j < qs.length; j++) { - Assert.assertTrue(j + "-th element org: " + VALUES[j] + ", act: " + vals[j], - Math.abs(vals[j] - VALUES[j]) <= 10); - } - hist.reset(); - } - } - - @Test - public void testUniform() { - FastLongHistogram hist = new FastLongHistogram(100, 0, 50); - doTestUniform(hist); - } - - @Test - public void testAdaptionOfChange() { - // assumes the uniform distribution - FastLongHistogram hist = new FastLongHistogram(100, 0, 100); - - Random rand = new Random(); - - for (int n = 0; n < 10; n++) { - for (int i = 0; i < 900; i++) { - hist.add(rand.nextInt(100), 1); - } - - // add 10% outliers, this breaks the assumption, hope bin10xMax works - for (int i = 0; i < 100; i++) { - hist.add(1000 + rand.nextInt(100), 1); - } - - long[] vals = hist.getQuantiles(new double[] { 0.25, 0.75, 0.95 }); - System.out.println(Arrays.toString(vals)); - if (n == 0) { - Assert.assertTrue("Out of possible value", vals[0] >= 0 && vals[0] <= 50); - Assert.assertTrue("Out of possible value", vals[1] >= 50 && vals[1] <= 100); - Assert.assertTrue("Out of possible value", vals[2] >= 900 && vals[2] <= 1100); - } - - hist.reset(); - } - } - - - @Test - public void testGetNumAtOrBelow() { - long[] VALUES = { 1, 10, 20, 30, 40, 50 }; - - FastLongHistogram h = new FastLongHistogram(); - for (long v : VALUES) { - for (int i = 0; i < 100; i++) { - h.add(v, 1); - } - } - - h.add(Integer.MAX_VALUE, 1); - - h.reset(); - - for (long v : VALUES) { - for (int i = 0; i < 100; i++) { - h.add(v, 1); - } - } - // Add something way out there to make sure it doesn't throw off the counts. - h.add(Integer.MAX_VALUE, 1); - - assertEquals(100, h.getNumAtOrBelow(1)); - assertEquals(200, h.getNumAtOrBelow(11)); - assertEquals(601, h.getNumAtOrBelow(Long.MAX_VALUE)); - } - - - @Test - public void testSameValues() { - FastLongHistogram hist = new FastLongHistogram(100); - - hist.add(50, 100); - - hist.reset(); - doTestUniform(hist); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleMasterObserverWithMetrics.java ---------------------------------------------------------------------- diff --git a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleMasterObserverWithMetrics.java b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleMasterObserverWithMetrics.java new file mode 100644 index 0000000..a5bae0a --- /dev/null +++ b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleMasterObserverWithMetrics.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor.example; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.metrics.Counter; +import org.apache.hadoop.hbase.metrics.Gauge; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.Timer; + +/** + * An example coprocessor that collects some metrics to demonstrate the usage of exporting custom + * metrics from the coprocessor. + * + *

+ * These metrics will be available through the regular Hadoop metrics2 sinks (ganglia, opentsdb, + * etc) as well as JMX output. You can view a snapshot of the metrics by going to the http web UI + * of the master page, something like http://mymasterhost:16010/jmx + *

+ * @see ExampleRegionObserverWithMetrics + */ +public class ExampleMasterObserverWithMetrics extends BaseMasterObserver { + + private static final Log LOG = LogFactory.getLog(ExampleMasterObserverWithMetrics.class); + + /** This is the Timer metric object to keep track of the current count across invocations */ + private Timer createTableTimer; + private long createTableStartTime = Long.MIN_VALUE; + + /** This is a Counter object to keep track of disableTable operations */ + private Counter disableTableCounter; + + /** Returns the total memory of the process. We will use this to define a gauge metric */ + private long getTotalMemory() { + return Runtime.getRuntime().totalMemory(); + } + + /** Returns the max memory of the process. We will use this to define a gauge metric */ + private long getMaxMemory() { + return Runtime.getRuntime().maxMemory(); + } + + @Override + public void preCreateTable(ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + super.preCreateTable(ctx, desc, regions); + // we rely on the fact that there is only 1 instance of our MasterObserver. We keep track of + // when the operation starts before the operation is executing. + this.createTableStartTime = System.currentTimeMillis(); + } + + @Override + public void postCreateTable(ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + super.postCreateTable(ctx, desc, regions); + if (this.createTableStartTime > 0) { + long time = System.currentTimeMillis() - this.createTableStartTime; + LOG.info("Create table took: " + time); + + // Update the timer metric for the create table operation duration. + createTableTimer.updateMillis(time); + } + } + + @Override + public void preDisableTable(ObserverContext ctx, TableName tableName) throws IOException { + super.preDisableTable(ctx, tableName); + + // Increment the Counter for disable table operations + this.disableTableCounter.increment(); + } + + @Override + public void start(CoprocessorEnvironment env) throws IOException { + super.start(env); + + // start for the MasterObserver will be called only once in the lifetime of the + // server. We will construct and register all metrics that we will track across method + // invocations. + + if (env instanceof MasterCoprocessorEnvironment) { + // Obtain the MetricRegistry for the Master. Metrics from this registry will be reported + // at the master level per-server. + MetricRegistry registry = + ((MasterCoprocessorEnvironment) env).getMetricRegistryForMaster(); + + if (createTableTimer == null) { + // Create a new Counter, or get the already registered counter. + // It is much better to only call this once and save the Counter as a class field instead + // of creating the counter every time a coprocessor method is invoked. This will negate + // any performance bottleneck coming from map lookups tracking metrics in the registry. + createTableTimer = registry.timer("CreateTable"); + + // on stop(), we can remove these registered metrics via calling registry.remove(). But + // it is not needed for coprocessors at the master level. If coprocessor is stopped, + // the server is stopping anyway, so there will not be any resource leaks. + } + + if (disableTableCounter == null) { + disableTableCounter = registry.counter("DisableTable"); + } + + // Register a custom gauge. The Gauge object will be registered in the metrics registry and + // periodically the getValue() is invoked to obtain the snapshot. + registry.register("totalMemory", new Gauge() { + @Override + public Long getValue() { + return getTotalMemory(); + } + }); + + // Register a custom gauge using Java-8 lambdas (Supplier converted into Gauge) + registry.register("maxMemory", this::getMaxMemory); + } + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleRegionObserverWithMetrics.java ---------------------------------------------------------------------- diff --git a/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleRegionObserverWithMetrics.java b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleRegionObserverWithMetrics.java new file mode 100644 index 0000000..7606b05 --- /dev/null +++ b/hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleRegionObserverWithMetrics.java @@ -0,0 +1,121 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.coprocessor.example; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.metrics.Counter; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.Timer; + +/** + * An example coprocessor that collects some metrics to demonstrate the usage of exporting custom + * metrics from the coprocessor. + *

+ * These metrics will be available through the regular Hadoop metrics2 sinks (ganglia, opentsdb, + * etc) as well as JMX output. You can view a snapshot of the metrics by going to the http web UI + * of the regionserver page, something like http://myregionserverhost:16030/jmx + *

+ * + * @see ExampleMasterObserverWithMetrics + */ +public class ExampleRegionObserverWithMetrics extends BaseRegionObserver { + + private Counter preGetCounter; + private Timer costlyOperationTimer; + + @Override + public void preGetOp(ObserverContext e, Get get, List results) + throws IOException { + super.preGetOp(e, get, results); + + // Increment the Counter whenever the coprocessor is called + preGetCounter.increment(); + } + + @Override + public void postGetOp(ObserverContext e, Get get, + List results) throws IOException { + super.postGetOp(e, get, results); + + // do a costly (high latency) operation which we want to measure how long it takes by + // using a Timer (which is a Meter and a Histogram). + long start = System.nanoTime(); + try { + performCostlyOperation(); + } finally { + costlyOperationTimer.updateNanos(System.nanoTime() - start); + } + } + + private void performCostlyOperation() { + try { + // simulate the operation by sleeping. + Thread.sleep(ThreadLocalRandom.current().nextLong(100)); + } catch (InterruptedException ignore) {} + } + + @Override + public void start(CoprocessorEnvironment env) throws IOException { + super.start(env); + + // start for the RegionServerObserver will be called only once in the lifetime of the + // server. We will construct and register all metrics that we will track across method + // invocations. + + if (env instanceof RegionCoprocessorEnvironment) { + // Obtain the MetricRegistry for the RegionServer. Metrics from this registry will be reported + // at the region server level per-regionserver. + MetricRegistry registry = + ((RegionCoprocessorEnvironment) env).getMetricRegistryForRegionServer(); + + if (preGetCounter == null) { + // Create a new Counter, or get the already registered counter. + // It is much better to only call this once and save the Counter as a class field instead + // of creating the counter every time a coprocessor method is invoked. This will negate + // any performance bottleneck coming from map lookups tracking metrics in the registry. + // Returned counter instance is shared by all coprocessors of the same class in the same + // region server. + preGetCounter = registry.counter("preGetRequests"); + } + + if (costlyOperationTimer == null) { + // Create a Timer to track execution times for the costly operation. + costlyOperationTimer = registry.timer("costlyOperation"); + } + } + } + + @Override + public void stop(CoprocessorEnvironment e) throws IOException { + // we should NOT remove / deregister the metrics in stop(). The whole registry will be + // removed when the last region of the table is closed. + super.stop(e); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-hadoop-compat/pom.xml ---------------------------------------------------------------------- diff --git a/hbase-hadoop-compat/pom.xml b/hbase-hadoop-compat/pom.xml index 016dd24..1ec1047 100644 --- a/hbase-hadoop-compat/pom.xml +++ b/hbase-hadoop-compat/pom.xml @@ -122,6 +122,10 @@ org.apache.commons commons-math + + org.apache.hbase + hbase-metrics-api + http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java index f79aa9f..652aae1 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java @@ -104,4 +104,9 @@ public interface BaseSource { */ String getMetricsName(); + default MetricRegistryInfo getMetricRegistryInfo() { + return new MetricRegistryInfo(getMetricsName(), getMetricsDescription(), + getMetricsContext(), getMetricsJmxContext(), true); + } + } http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-hadoop2-compat/pom.xml ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/pom.xml b/hbase-hadoop2-compat/pom.xml index 56c3e7f..3e63391 100644 --- a/hbase-hadoop2-compat/pom.xml +++ b/hbase-hadoop2-compat/pom.xml @@ -166,6 +166,10 @@ limitations under the License. org.apache.hbase + hbase-metrics + + + org.apache.hbase hbase-hadoop-compat ${project.version} test-jar http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java index f843ec2..6a9fff4 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java @@ -19,6 +19,9 @@ package org.apache.hadoop.hbase.metrics; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.metrics.impl.GlobalMetricRegistriesAdapter; +import org.apache.hadoop.hbase.metrics.impl.HBaseMetrics2HadoopMetricsAdapter; +import org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceImpl; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsSource; import org.apache.hadoop.metrics2.impl.JmxCacheBuster; @@ -47,15 +50,53 @@ public class BaseSourceImpl implements BaseSource, MetricsSource { inited = true; DefaultMetricsSystem.initialize(HBASE_METRICS_SYSTEM_NAME); JvmMetrics.initSingleton(name, ""); + // initialize hbase-metrics module based metric system as well. GlobalMetricRegistriesSource + // initialization depends on the metric system being already initialized, that is why we are + // doing it here. Once BaseSourceSourceImpl is removed, we should do the initialization of + // these elsewhere. + GlobalMetricRegistriesAdapter.init(); } } + /** + * @deprecated Use hbase-metrics/hbase-metrics-api module interfaces for new metrics. + * Defining BaseSources for new metric groups (WAL, RPC, etc) is not needed anymore, however, + * for existing BaseSource implemetnations, please use the field named "registry" which is a + * MetricRegistry instance together with the HBaseMetrics2HadoopMetricsAdapter. + */ + @Deprecated protected final DynamicMetricsRegistry metricsRegistry; protected final String metricsName; protected final String metricsDescription; protected final String metricsContext; protected final String metricsJmxContext; + /** + * Note that there are at least 4 MetricRegistry definitions in the source code. The first one is + * Hadoop Metrics2 MetricRegistry, second one is DynamicMetricsRegistry which is HBase's fork + * of the Hadoop metrics2 class. The third one is the dropwizard metrics implementation of + * MetricRegistry, and finally a new API abstraction in HBase that is the + * o.a.h.h.metrics.MetricRegistry class. This last one is the new way to use metrics within the + * HBase code. However, the others are in play because of existing metrics2 based code still + * needs to coexists until we get rid of all of our BaseSource and convert them to the new + * framework. Until that happens, new metrics can use the new API, but will be collected + * through the HBaseMetrics2HadoopMetricsAdapter class. + * + * BaseSourceImpl has two MetricRegistries. metricRegistry is for hadoop Metrics2 based + * metrics, while the registry is for hbase-metrics based metrics. + */ + protected final MetricRegistry registry; + + /** + * The adapter from hbase-metrics module to metrics2. This adepter is the connection between the + * Metrics in the MetricRegistry and the Hadoop Metrics2 system. Using this adapter, existing + * BaseSource implementations can define new metrics using the hbase-metrics/hbase-metrics-api + * module interfaces and still be able to make use of metrics2 sinks (including JMX). Existing + * BaseSources should call metricsAdapter.snapshotAllMetrics() in getMetrics() method. See + * {@link MetricsRegionServerSourceImpl}. + */ + protected final HBaseMetrics2HadoopMetricsAdapter metricsAdapter; + public BaseSourceImpl( String metricsName, String metricsDescription, @@ -72,6 +113,11 @@ public class BaseSourceImpl implements BaseSource, MetricsSource { //Register this instance. DefaultMetricsSystem.instance().register(metricsJmxContext, metricsDescription, this); + + // hbase-metrics module based metrics are registered in the hbase MetricsRegistry. + registry = MetricRegistries.global().create(this.getMetricRegistryInfo()); + metricsAdapter = new HBaseMetrics2HadoopMetricsAdapter(); + init(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/impl/GlobalMetricRegistriesAdapter.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/impl/GlobalMetricRegistriesAdapter.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/impl/GlobalMetricRegistriesAdapter.java new file mode 100644 index 0000000..ddcf56d --- /dev/null +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/impl/GlobalMetricRegistriesAdapter.java @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + + +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.Optional; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.metrics.MetricRegistries; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsExecutor; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.impl.JmxCacheBuster; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystemHelper; +import org.apache.hadoop.metrics2.lib.MetricsExecutorImpl; + +import com.google.common.annotations.VisibleForTesting; + +/** + * This class acts as an adapter to export the MetricRegistry's in the global registry. Each + * MetricRegistry will be registered or unregistered from the metric2 system. The collection will + * be performed via the MetricsSourceAdapter and the MetricRegistry will collected like a + * BaseSource instance for a group of metrics (like WAL, RPC, etc) with the MetricRegistryInfo's + * JMX context. + * + *

Developer note: + * Unlike the current metrics2 based approach, the new metrics approach + * (hbase-metrics-api and hbase-metrics modules) work by having different MetricRegistries that are + * initialized and used from the code that lives in their respective modules (hbase-server, etc). + * There is no need to define BaseSource classes and do a lot of indirection. The MetricRegistry'es + * will be in the global MetricRegistriesImpl, and this class will iterate over + * MetricRegistries.global() and register adapters to the metrics2 subsystem. These adapters then + * report the actual values by delegating to + * {@link HBaseMetrics2HadoopMetricsAdapter#snapshotAllMetrics(MetricRegistry, MetricsCollector)}. + * + * We do not initialize the Hadoop Metrics2 system assuming that other BaseSources already do so + * (see BaseSourceImpl). Once the last BaseSource is moved to the new system, the metric2 + * initialization should be moved here. + *

+ */ +public class GlobalMetricRegistriesAdapter { + + private static final Log LOG = LogFactory.getLog(GlobalMetricRegistriesAdapter.class); + + private class MetricsSourceAdapter implements MetricsSource { + private final MetricRegistry registry; + MetricsSourceAdapter(MetricRegistry registry) { + this.registry = registry; + } + + @Override + public void getMetrics(MetricsCollector collector, boolean all) { + metricsAdapter.snapshotAllMetrics(registry, collector); + } + } + + private final MetricsExecutor executor; + private final AtomicBoolean stopped; + private final DefaultMetricsSystemHelper helper; + private final HBaseMetrics2HadoopMetricsAdapter metricsAdapter; + private final HashMap registeredSources; + + private GlobalMetricRegistriesAdapter() { + this.executor = new MetricsExecutorImpl(); + this.stopped = new AtomicBoolean(false); + this.metricsAdapter = new HBaseMetrics2HadoopMetricsAdapter(); + this.registeredSources = new HashMap<>(); + this.helper = new DefaultMetricsSystemHelper(); + executor.getExecutor().scheduleAtFixedRate(() -> this.doRun(), 10, 10, TimeUnit.SECONDS); + } + + /** + * Make sure that this global MetricSource for hbase-metrics module based metrics are initialized. + * This should be called only once. + */ + public static GlobalMetricRegistriesAdapter init() { + return new GlobalMetricRegistriesAdapter(); + } + + @VisibleForTesting + public void stop() { + stopped.set(true); + } + + private void doRun() { + if (stopped.get()) { + executor.stop(); + return; + } + if (LOG.isTraceEnabled()) { + LOG.trace("doRun called: " + registeredSources); + } + + Collection registries = MetricRegistries.global().getMetricRegistries(); + for (MetricRegistry registry : registries) { + MetricRegistryInfo info = registry.getMetricRegistryInfo(); + + if (info.isExistingSource()) { + // If there is an already existing BaseSource for this MetricRegistry, skip it here. These + // types of registries are there only due to existing BaseSource implementations in the + // source code (like MetricsRegionServer, etc). This is to make sure that we can transition + // iteratively to the new hbase-metrics system. These type of MetricRegistry metrics will be + // exported from the BaseSource.getMetrics() call directly because there is already a + // MetricRecordBuilder there (see MetricsRegionServerSourceImpl). + continue; + } + + if (!registeredSources.containsKey(info)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Registering adapter for the MetricRegistry: " + info.getMetricsJmxContext()); + } + // register this as a MetricSource under different JMX Context'es. + MetricsSourceAdapter adapter = new MetricsSourceAdapter(registry); + LOG.info("Registering " + info.getMetricsJmxContext() + " " + info.getMetricsDescription()); + DefaultMetricsSystem.instance().register(info.getMetricsJmxContext(), + info.getMetricsDescription(), adapter); + registeredSources.put(info, adapter); + // next collection will collect the newly registered MetricSource. Doing this here leads to + // ConcurrentModificationException. + } + } + + boolean removed = false; + // Remove registered sources if it is removed from the global registry + for (Iterator> it = + registeredSources.entrySet().iterator(); it.hasNext();) { + Entry entry = it.next(); + MetricRegistryInfo info = entry.getKey(); + Optional found = MetricRegistries.global().get(info); + if (!found.isPresent()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Removing adapter for the MetricRegistry: " + info.getMetricsJmxContext()); + } + synchronized(DefaultMetricsSystem.instance()) { + DefaultMetricsSystem.instance().unregisterSource(info.getMetricsJmxContext()); + helper.removeSourceName(info.getMetricsJmxContext()); + helper.removeObjectName(info.getMetricsJmxContext()); + it.remove(); + removed = true; + } + } + } + if (removed) { + JmxCacheBuster.clearJmxCache(); + } + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/impl/HBaseMetrics2HadoopMetricsAdapter.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/impl/HBaseMetrics2HadoopMetricsAdapter.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/impl/HBaseMetrics2HadoopMetricsAdapter.java new file mode 100644 index 0000000..ec4a1a7 --- /dev/null +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/impl/HBaseMetrics2HadoopMetricsAdapter.java @@ -0,0 +1,169 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Copyright 2016 Josh Elser + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.metrics.Counter; +import org.apache.hadoop.hbase.metrics.Gauge; +import org.apache.hadoop.hbase.metrics.Histogram; +import org.apache.hadoop.hbase.metrics.Meter; +import org.apache.hadoop.hbase.metrics.Metric; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; +import org.apache.hadoop.hbase.metrics.Timer; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.lib.Interns; +import org.apache.hadoop.metrics2.lib.MutableHistogram; + +/** + * This is the adapter from "HBase Metrics Framework", implemented in hbase-metrics-api and + * hbase-metrics modules to the Hadoop Metrics2 framework. This adapter is not a metric source, + * but a helper to be able to collect all of the Metric's in the MetricRegistry using the + * MetricsCollector and MetricsRecordBuilder. + * + * Some of the code is forked from https://github.com/joshelser/dropwizard-hadoop-metrics2. + */ +public class HBaseMetrics2HadoopMetricsAdapter { + private static final Log LOG + = LogFactory.getLog(HBaseMetrics2HadoopMetricsAdapter.class); + private static final String EMPTY_STRING = ""; + + public HBaseMetrics2HadoopMetricsAdapter() { + } + + /** + * Iterates over the MetricRegistry and adds them to the {@code collector}. + * + * @param collector A metrics collector + */ + public void snapshotAllMetrics(MetricRegistry metricRegistry, + MetricsCollector collector) { + MetricRegistryInfo info = metricRegistry.getMetricRegistryInfo(); + MetricsRecordBuilder builder = collector.addRecord(Interns.info(info.getMetricsName(), + info.getMetricsDescription())); + builder.setContext(info.getMetricsContext()); + + snapshotAllMetrics(metricRegistry, builder); + } + + /** + * Iterates over the MetricRegistry and adds them to the {@code builder}. + * + * @param builder A record builder + */ + public void snapshotAllMetrics(MetricRegistry metricRegistry, MetricsRecordBuilder builder) { + Map metrics = metricRegistry.getMetrics(); + + for (Map.Entry e: metrics.entrySet()) { + // Always capitalize the name + String name = StringUtils.capitalize(e.getKey()); + Metric metric = e.getValue(); + + if (metric instanceof Gauge) { + addGauge(name, (Gauge) metric, builder); + } else if (metric instanceof Counter) { + addCounter(name, (Counter)metric, builder); + } else if (metric instanceof Histogram) { + addHistogram(name, (Histogram)metric, builder); + } else if (metric instanceof Meter) { + addMeter(name, (Meter)metric, builder); + } else if (metric instanceof Timer) { + addTimer(name, (Timer)metric, builder); + } else { + LOG.info("Ignoring unknown Metric class " + metric.getClass().getName()); + } + } + } + + private void addGauge(String name, Gauge gauge, MetricsRecordBuilder builder) { + final MetricsInfo info = Interns.info(name, EMPTY_STRING); + final Object o = gauge.getValue(); + + // Figure out which gauge types metrics2 supports and call the right method + if (o instanceof Integer) { + builder.addGauge(info, (int) o); + } else if (o instanceof Long) { + builder.addGauge(info, (long) o); + } else if (o instanceof Float) { + builder.addGauge(info, (float) o); + } else if (o instanceof Double) { + builder.addGauge(info, (double) o); + } else { + LOG.warn("Ignoring Gauge (" + name + ") with unhandled type: " + o.getClass()); + } + } + + private void addCounter(String name, Counter counter, MetricsRecordBuilder builder) { + MetricsInfo info = Interns.info(name, EMPTY_STRING); + builder.addCounter(info, counter.getCount()); + } + + /** + * Add Histogram value-distribution data to a Hadoop-Metrics2 record building. + * + * @param builder A Hadoop-Metrics2 record builder. + * @param name A base name for this record. + * @param desc A description for this record. + * @param snapshot The distribution of measured values. + */ + private void addHistogram(String name, Histogram histogram, MetricsRecordBuilder builder) { + MutableHistogram.snapshot(name, EMPTY_STRING, histogram, builder, true); + } + + /** + * Add Dropwizard-Metrics rate information to a Hadoop-Metrics2 record builder, converting the + * rates to the appropriate unit. + * + * @param builder A Hadoop-Metrics2 record builder. + * @param name A base name for this record. + */ + private void addMeter(String name, Meter meter, MetricsRecordBuilder builder) { + builder.addGauge(Interns.info(name + "_count", EMPTY_STRING), meter.getCount()); + builder.addGauge(Interns.info(name + "_mean_rate", EMPTY_STRING), meter.getMeanRate()); + builder.addGauge(Interns.info(name + "_1min_rate", EMPTY_STRING), meter.getOneMinuteRate()); + builder.addGauge(Interns.info(name + "_5min_rate", EMPTY_STRING), meter.getFiveMinuteRate()); + builder.addGauge(Interns.info(name + "_15min_rate", EMPTY_STRING), + meter.getFifteenMinuteRate()); + } + + private void addTimer(String name, Timer timer, MetricsRecordBuilder builder) { + addMeter(name, timer.getMeter(), builder); + addHistogram(name, timer.getHistogram(), builder); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index b412fd1..67764f8 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -510,6 +510,12 @@ public class MetricsRegionServerSourceImpl } metricsRegistry.snapshot(mrb, all); + + // source is registered in supers constructor, sometimes called before the whole initialization. + if (metricsAdapter != null) { + // snapshot MetricRegistry as well + metricsAdapter.snapshotAllMetrics(registry, mrb); + } } @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystemHelper.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystemHelper.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystemHelper.java index 832e220..eb465c3 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystemHelper.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystemHelper.java @@ -17,26 +17,44 @@ */ package org.apache.hadoop.metrics2.lib; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.HashMap; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import java.lang.reflect.Method; - public class DefaultMetricsSystemHelper { private static final Log LOG = LogFactory.getLog(DefaultMetricsSystemHelper.class); private final Method removeObjectMethod; + private final Field sourceNamesField; + private final Field mapField; public DefaultMetricsSystemHelper() { + Class clazz = DefaultMetricsSystem.INSTANCE.getClass(); Method m; try { - Class clazz = DefaultMetricsSystem.INSTANCE.getClass(); m = clazz.getDeclaredMethod("removeObjectName", String.class); m.setAccessible(true); } catch (NoSuchMethodException e) { m = null; } removeObjectMethod = m; + + Field f1, f2; + try { + f1 = clazz.getDeclaredField("sourceNames"); + f1.setAccessible(true); + f2 = UniqueNames.class.getDeclaredField("map"); + f2.setAccessible(true); + } catch (NoSuchFieldException e) { + LOG.trace(e); + f1 = null; + f2 = null; + } + sourceNamesField = f1; + mapField = f2; } public boolean removeObjectName(final String name) { @@ -52,4 +70,30 @@ public class DefaultMetricsSystemHelper { } return false; } + + /** + * Unfortunately Hadoop tries to be too-clever and permanently keeps track of all names registered + * so far as a Source, thus preventing further re-registration of the source with the same name. + * In case of dynamic metrics tied to region-lifecycles, this becomes a problem because we would + * like to be able to re-register and remove with the same name. Otherwise, it is resource leak. + * This ugly code manually removes the name from the UniqueNames map. + * TODO: May not be needed for Hadoop versions after YARN-5190. + */ + public void removeSourceName(String name) { + if (sourceNamesField == null || mapField == null) { + return; + } + try { + Object sourceNames = sourceNamesField.get(DefaultMetricsSystem.INSTANCE); + HashMap map = (HashMap) mapField.get(sourceNames); + synchronized (sourceNames) { + map.remove(name); + } + } catch (Exception ex) { + if (LOG.isTraceEnabled()) { + LOG.trace("Received exception while trying to access Hadoop Metrics classes via reflection.", + ex); + } + } + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java index ec8cb50..6a2f203 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java @@ -18,12 +18,12 @@ package org.apache.hadoop.metrics2.lib; -import java.util.concurrent.atomic.LongAdder; - import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.metrics.Histogram; import org.apache.hadoop.hbase.metrics.Interns; -import org.apache.hadoop.hbase.util.FastLongHistogram; +import org.apache.hadoop.hbase.metrics.Snapshot; +import org.apache.hadoop.hbase.metrics.impl.HistogramImpl; import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.MetricsRecordBuilder; @@ -33,28 +33,9 @@ import org.apache.hadoop.metrics2.MetricsRecordBuilder; */ @InterfaceAudience.Private public class MutableHistogram extends MutableMetric implements MetricHistogram { - // Double buffer the two FastLongHistograms. - // As they are reset they learn how the buckets should be spaced - // So keep two around and use them - protected final FastLongHistogram histogram; - + protected HistogramImpl histogram; protected final String name; protected final String desc; - protected final LongAdder counter = new LongAdder(); - - private boolean metricsInfoStringInited = false; - private String NUM_OPS_METRIC; - private String MIN_METRIC; - private String MAX_METRIC; - private String MEAN_METRIC; - private String MEDIAN_METRIC; - private String TWENTY_FIFTH_PERCENTILE_METRIC; - private String SEVENTY_FIFTH_PERCENTILE_METRIC; - private String NINETIETH_PERCENTILE_METRIC; - private String NINETY_FIFTH_PERCENTILE_METRIC; - private String NINETY_EIGHTH_PERCENTILE_METRIC; - private String NINETY_NINETH_PERCENTILE_METRIC; - private String NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC; public MutableHistogram(MetricsInfo info) { this(info.name(), info.description()); @@ -67,12 +48,11 @@ public class MutableHistogram extends MutableMetric implements MetricHistogram { protected MutableHistogram(String name, String description, long maxExpected) { this.name = StringUtils.capitalize(name); this.desc = StringUtils.uncapitalize(description); - this.histogram = new FastLongHistogram(FastLongHistogram.DEFAULT_NBINS, 1, maxExpected); + this.histogram = new HistogramImpl(); } public void add(final long val) { - counter.increment(); - histogram.add(val, 1); + histogram.update(val); } public long getMax() { @@ -81,56 +61,42 @@ public class MutableHistogram extends MutableMetric implements MetricHistogram { @Override public synchronized void snapshot(MetricsRecordBuilder metricsRecordBuilder, boolean all) { - // Get a reference to the old histogram. - FastLongHistogram histo = histogram.reset(); - if (histo != null) { - updateSnapshotMetrics(metricsRecordBuilder, histo); - } + snapshot(name, desc, histogram, metricsRecordBuilder, all); } - protected void updateSnapshotMetrics(MetricsRecordBuilder metricsRecordBuilder, - FastLongHistogram histo) { - if (!metricsInfoStringInited) { - NUM_OPS_METRIC = name + NUM_OPS_METRIC_NAME; - MIN_METRIC = name + MIN_METRIC_NAME; - MAX_METRIC = name + MAX_METRIC_NAME; - MEAN_METRIC = name + MEAN_METRIC_NAME; - MEDIAN_METRIC = name + MEDIAN_METRIC_NAME; - TWENTY_FIFTH_PERCENTILE_METRIC = name + TWENTY_FIFTH_PERCENTILE_METRIC_NAME; - SEVENTY_FIFTH_PERCENTILE_METRIC = name + SEVENTY_FIFTH_PERCENTILE_METRIC_NAME; - NINETIETH_PERCENTILE_METRIC = name + NINETIETH_PERCENTILE_METRIC_NAME; - NINETY_FIFTH_PERCENTILE_METRIC = name + NINETY_FIFTH_PERCENTILE_METRIC_NAME; - NINETY_EIGHTH_PERCENTILE_METRIC = name + NINETY_EIGHTH_PERCENTILE_METRIC_NAME; - NINETY_NINETH_PERCENTILE_METRIC = name + NINETY_NINETH_PERCENTILE_METRIC_NAME; - NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC = name + - NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC_NAME; - - metricsInfoStringInited = true; + public static void snapshot(String name, String desc, Histogram histogram, + MetricsRecordBuilder metricsRecordBuilder, boolean all) { + // Get a reference to the old histogram. + Snapshot snapshot = histogram.snapshot(); + if (snapshot != null) { + updateSnapshotMetrics(name, desc, histogram, snapshot, metricsRecordBuilder); } + } - metricsRecordBuilder.addCounter(Interns.info(NUM_OPS_METRIC, desc), counter.sum()); - metricsRecordBuilder.addGauge(Interns.info(MIN_METRIC, desc), histo.getMin()); - metricsRecordBuilder.addGauge(Interns.info(MAX_METRIC, desc), histo.getMax()); - metricsRecordBuilder.addGauge(Interns.info(MEAN_METRIC, desc), histo.getMean()); - - long[] percentiles = histo.getQuantiles(); - - metricsRecordBuilder.addGauge(Interns.info(TWENTY_FIFTH_PERCENTILE_METRIC, desc), - percentiles[0]); - metricsRecordBuilder.addGauge(Interns.info(MEDIAN_METRIC, desc), - percentiles[1]); - metricsRecordBuilder.addGauge(Interns.info(SEVENTY_FIFTH_PERCENTILE_METRIC, desc), - percentiles[2]); - metricsRecordBuilder.addGauge(Interns.info(NINETIETH_PERCENTILE_METRIC, desc), - percentiles[3]); - metricsRecordBuilder.addGauge(Interns.info(NINETY_FIFTH_PERCENTILE_METRIC, desc), - percentiles[4]); - metricsRecordBuilder.addGauge(Interns.info(NINETY_EIGHTH_PERCENTILE_METRIC, desc), - percentiles[5]); - metricsRecordBuilder.addGauge(Interns.info(NINETY_NINETH_PERCENTILE_METRIC, desc), - percentiles[6]); + protected static void updateSnapshotMetrics(String name, String desc, Histogram histogram, + Snapshot snapshot, MetricsRecordBuilder metricsRecordBuilder) { + metricsRecordBuilder.addCounter(Interns.info(name + NUM_OPS_METRIC_NAME, desc), + histogram.getCount()); + metricsRecordBuilder.addGauge(Interns.info(name + MIN_METRIC_NAME, desc), snapshot.getMin()); + metricsRecordBuilder.addGauge(Interns.info(name + MAX_METRIC_NAME, desc), snapshot.getMax()); + metricsRecordBuilder.addGauge(Interns.info(name + MEAN_METRIC_NAME, desc), snapshot.getMean()); + + metricsRecordBuilder.addGauge(Interns.info(name + TWENTY_FIFTH_PERCENTILE_METRIC_NAME, desc), + snapshot.get25thPercentile()); + metricsRecordBuilder.addGauge(Interns.info(name + MEDIAN_METRIC_NAME, desc), + snapshot.getMedian()); + metricsRecordBuilder.addGauge(Interns.info(name + SEVENTY_FIFTH_PERCENTILE_METRIC_NAME, desc), + snapshot.get75thPercentile()); + metricsRecordBuilder.addGauge(Interns.info(name + NINETIETH_PERCENTILE_METRIC_NAME, desc), + snapshot.get90thPercentile()); + metricsRecordBuilder.addGauge(Interns.info(name + NINETY_FIFTH_PERCENTILE_METRIC_NAME, desc), + snapshot.get95thPercentile()); + metricsRecordBuilder.addGauge(Interns.info(name + NINETY_EIGHTH_PERCENTILE_METRIC_NAME, desc), + snapshot.get98thPercentile()); + metricsRecordBuilder.addGauge(Interns.info(name + NINETY_NINETH_PERCENTILE_METRIC_NAME, desc), + snapshot.get99thPercentile()); metricsRecordBuilder.addGauge( - Interns.info(NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC, desc), - percentiles[7]); + Interns.info(name + NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC_NAME, desc), + snapshot.get999thPercentile()); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java ---------------------------------------------------------------------- diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java index 94bcdaa..a12dc27 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java @@ -20,7 +20,7 @@ package org.apache.hadoop.metrics2.lib; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.Interns; -import org.apache.hadoop.hbase.util.FastLongHistogram; +import org.apache.hadoop.hbase.metrics.Snapshot; import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.MetricsRecordBuilder; @@ -56,22 +56,22 @@ public abstract class MutableRangeHistogram extends MutableHistogram implements @Override public synchronized void snapshot(MetricsRecordBuilder metricsRecordBuilder, boolean all) { // Get a reference to the old histogram. - FastLongHistogram histo = histogram.reset(); - if (histo != null) { - updateSnapshotMetrics(metricsRecordBuilder, histo); - updateSnapshotRangeMetrics(metricsRecordBuilder, histo); + Snapshot snapshot = histogram.snapshot(); + if (snapshot != null) { + updateSnapshotMetrics(name, desc, histogram, snapshot, metricsRecordBuilder); + updateSnapshotRangeMetrics(metricsRecordBuilder, snapshot); } } public void updateSnapshotRangeMetrics(MetricsRecordBuilder metricsRecordBuilder, - FastLongHistogram histogram) { + Snapshot snapshot) { long priorRange = 0; long cumNum = 0; final long[] ranges = getRanges(); final String rangeType = getRangeType(); for (int i = 0; i < ranges.length - 1; i++) { - long val = histogram.getNumAtOrBelow(ranges[i]); + long val = snapshot.getCountAtOrBelow(ranges[i]); if (val - cumNum > 0) { metricsRecordBuilder.addCounter( Interns.info(name + "_" + rangeType + "_" + priorRange + "-" + ranges[i], desc), http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-metrics-api/README.txt ---------------------------------------------------------------------- diff --git a/hbase-metrics-api/README.txt b/hbase-metrics-api/README.txt new file mode 100644 index 0000000..dfaa29f --- /dev/null +++ b/hbase-metrics-api/README.txt @@ -0,0 +1,78 @@ +Overview +======== +hbase-metrics and hbase-metrics-api are two modules that define and implement the "new" metric +system used internally within HBase. These two modules (and some other code in hbase-hadoop2-compat) +module are referred as "HBase metrics framework". + +HBase-metrics-api Module +======================== +HBase Metrics API (hbase-metrics-api) contains the interface +that HBase exposes internally and to third party code (including coprocessors). It is a thin +abstraction over the actual implementation for backwards compatibility guarantees. The source +/ binary and other compatibility guarantees are for "LimitedPrivate API" (see [1] for an +explanation). + +The metrics API in this hbase-metrics-api module is inspired by the Dropwizard metrics 3.1 API +(See [2]). It is a subset of the API only containing metrics collection. However, the implementation +is HBase-specific and provided in hbase-metrics module. All of the classes in this module is +HBase-internal. See the latest documentation of Dropwizard metrics for examples of defining / using +metrics. + + +HBase-metrics Module +==================== +hbase-metrics module contains implementation of the "HBase Metrics API", including MetricRegistry, +Counter, Histogram, etc. These are highly concurrent implementations of the Metric interfaces. +Metrics in HBase are grouped into different sets (like WAL, RPC, RegionServer, etc). Each group of +metrics should be tracked via a MetricRegistry specific to that group. Metrics can be dynamically +added or removed from the registry with a name. Each Registry is independent of the other +registries and will have it's own JMX context and MetricRecord (when used with Metrics2). + + +MetricRegistry's themselves are tracked via a global registry (of MetricRegistries) called +MetricRegistries. MetricRegistries.global() can be used to obtain the global instance. +MetricRegistry instances can also be dynamically registered and removed. However, unlike the +MetricRegistry, MetricRegistries does reference counting of the MetricRegistry instances. Only +Metrics in the MetricRegistry instances that are in the global MetricRegistry are exported to the +metric sinks or JMX. + + +Coprocessor Metrics +=================== +HBase allows custom coprocessors to track and export metrics using the new framework. +Coprocessors and other third party code should only use the classes and interfaces from +hbase-metrics-api module and only the classes that are marked with InterfaceAudience.LimitedPrivate +annotation. There is no guarantee on the compatibility requirements for other classes. + +Coprocessors can obtain the MetricRegistry to register their custom metrics via corresponding +CoprocessorEnvironment context. See ExampleRegionObserverWithMetrics and +ExampleMasterObserverWithMetrics classes in hbase-examples module for usage. + + +Developer Notes +=============== +Historically, HBase has been using Hadoop's Metrics2 framework [3] for collecting and reporting the +metrics internally. However, due to the difficultly of dealing with the Metrics2 framework, HBase is +moving away from Hadoop's metrics implementation to its custom implementation. The move will happen +incrementally, and during the time, both Hadoop Metrics2-based metrics and hbase-metrics module +based classes will be in the source code. All new implementations for metrics SHOULD use the new +API and framework. + +Examples of the new framework can be found in MetricsCoprocessor and MetricsRegionServerSourceImpl +classes. See HBASE-9774 [4] for more context. + +hbase-metrics module right now only deals with metrics tracking and collection. It does not do JMX +reporting or reporting to console, ganglia, opentsdb, etc. We use Hadoop's Metrics2 for reporting +metrics to different sinks or exporting via JMX. However, this is contained within the +hbase-hadoop2-compat module completely, so that rest of the code does not know anything about the +Metrics2 dependency. HBaseMetrics2HadoopMetricsAdapter is the adapter that can collect metrics +in a MetricRegistry using the metric2 MetricsCollector / MetricRecordBuilder interfaces. +GlobalMetricRegistriesSource is the global Metrics2 Source that collects all of the metrics in all +of the metric registries in the MetricRegistries.global() instance. + + +References +1. https://hbase.apache.org/book.html#hbase.versioning +2. http://metrics.dropwizard.io/ +3. https://hadoop.apache.org/docs/r2.7.2/api/org/apache/hadoop/metrics2/package-summary.html +4. https://issues.apache.org/jira/browse/HBASE-9774 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-metrics-api/pom.xml ---------------------------------------------------------------------- diff --git a/hbase-metrics-api/pom.xml b/hbase-metrics-api/pom.xml new file mode 100644 index 0000000..5822971 --- /dev/null +++ b/hbase-metrics-api/pom.xml @@ -0,0 +1,112 @@ + + + + 4.0.0 + + hbase + org.apache.hbase + 2.0.0-SNAPSHOT + .. + + + hbase-metrics-api + Apache HBase - Metrics API + HBase Metrics API descriptions + + + + + org.apache.maven.plugins + maven-site-plugin + + true + + + + + org.apache.maven.plugins + maven-source-plugin + + + + maven-assembly-plugin + ${maven.assembly.version} + + true + + + + maven-surefire-plugin + + + + secondPartTestsExecution + test + + test + + + true + + + + + + + + + + + org.apache.hbase + hbase-annotations + + + org.apache.hbase + hbase-annotations + test-jar + test + + + org.apache.hbase + hbase-common + + + + commons-logging + commons-logging + + + + + + + skip-metrics-api-tests + + + skip-metrics-api-tests + + + + true + + + + http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Counter.java ---------------------------------------------------------------------- diff --git a/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Counter.java b/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Counter.java new file mode 100644 index 0000000..2e4147e --- /dev/null +++ b/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Counter.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics; + +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * A mutable number optimized for high concurrency counting. + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) +@InterfaceStability.Evolving +public interface Counter extends Metric { + + /** + * Increment {@code this} by 1. + */ + void increment(); + + /** + * Increment {@code this} by {@code n}. + * + * @param n The amount to increment. + */ + void increment(long n); + + /** + * Decrement {@code this} by 1. + */ + void decrement(); + + /** + * Decrement {@code this} by {@code n}. + * + * @param n The amount to decrement. + */ + void decrement(long n); + + /** + * Returns the current value. + * @return the current value. + */ + long getCount(); +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Gauge.java ---------------------------------------------------------------------- diff --git a/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Gauge.java b/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Gauge.java new file mode 100644 index 0000000..90df8e0 --- /dev/null +++ b/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Gauge.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics; + +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * A metrics which measures a discrete value. + * + * @param The value of the Gauge. + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) +@InterfaceStability.Evolving +public interface Gauge extends Metric { + + T getValue(); + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/c64a1d19/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Histogram.java ---------------------------------------------------------------------- diff --git a/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Histogram.java b/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Histogram.java new file mode 100644 index 0000000..b5b54c7 --- /dev/null +++ b/hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Histogram.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics; + +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * A metric which measures the distribution of values. + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) +@InterfaceStability.Evolving +public interface Histogram extends Metric { + + /** + * Adds a new value to the distribution. + * + * @param value The value to add + */ + void update(int value); + + /** + * Adds a new value to the distribution. + * + * @param value The value to add + */ + void update(long value); + + /** + * Return the total number of values added to the histogram. + * @return the total number of values. + */ + long getCount(); + + /** + * Snapshot the current values in the Histogram + * @return a Snapshot of the distribution. + */ + @InterfaceAudience.Private + Snapshot snapshot(); + +}