bookkeeper-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From si...@apache.org
Subject bookkeeper git commit: BOOKKEEPER-1067: Add Prometheus stats provider
Date Mon, 15 May 2017 19:25:23 GMT
Repository: bookkeeper
Updated Branches:
  refs/heads/master 74f795136 -> d86351371


BOOKKEEPER-1067: Add Prometheus stats provider

Prometheus (https://prometheus.io) is a metrics collection system, similar but much more flexible
than graphite.

It would be good to expose the Bookie and BookKeeper client stats directly so that a Prometheus
instance can collect them (and also check the process status and add alerts).

Author: Matteo Merli <mmerli@apache.org>

Reviewers: Enrico Olivelli <eolivelli@gmail.com>, Sijie Guo <sijie@apache.org>

Closes #154 from merlimat/prometheus


Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo
Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/d8635137
Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/d8635137
Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/d8635137

Branch: refs/heads/master
Commit: d863513719b696638e3369f4bbfde9abf5fd3d5f
Parents: 74f7951
Author: Matteo Merli <mmerli@apache.org>
Authored: Mon May 15 12:25:18 2017 -0700
Committer: Sijie Guo <sijie@apache.org>
Committed: Mon May 15 12:25:18 2017 -0700

----------------------------------------------------------------------
 bookkeeper-benchmark/pom.xml                    |  4 +-
 bookkeeper-server/pom.xml                       |  4 +-
 bookkeeper-stats-providers/pom.xml              |  3 +-
 .../prometheus-metrics-provider/pom.xml         | 74 +++++++++++++++++
 .../bookkeeper/stats/PrometheusCounter.java     | 56 +++++++++++++
 .../stats/PrometheusMetricsProvider.java        | 83 ++++++++++++++++++++
 .../stats/PrometheusOpStatsLogger.java          | 79 +++++++++++++++++++
 .../bookkeeper/stats/PrometheusStatsLogger.java | 74 +++++++++++++++++
 bookkeeper-stats/pom.xml                        |  7 +-
 pom.xml                                         |  1 +
 10 files changed, 373 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/d8635137/bookkeeper-benchmark/pom.xml
----------------------------------------------------------------------
diff --git a/bookkeeper-benchmark/pom.xml b/bookkeeper-benchmark/pom.xml
index 4354656..d849fb2 100644
--- a/bookkeeper-benchmark/pom.xml
+++ b/bookkeeper-benchmark/pom.xml
@@ -61,12 +61,12 @@
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
-      <version>1.6.4</version>
+      <version>${slf4j.version}</version>
     </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-log4j12</artifactId>
-      <version>1.6.4</version>
+      <version>${slf4j.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.zookeeper</groupId>

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/d8635137/bookkeeper-server/pom.xml
----------------------------------------------------------------------
diff --git a/bookkeeper-server/pom.xml b/bookkeeper-server/pom.xml
index 99705f4..e5de842 100644
--- a/bookkeeper-server/pom.xml
+++ b/bookkeeper-server/pom.xml
@@ -56,12 +56,12 @@
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
-      <version>1.7.25</version>
+      <version>${slf4j.version}</version>
     </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-log4j12</artifactId>
-      <version>1.7.25</version>
+      <version>${slf4j.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.zookeeper</groupId>

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/d8635137/bookkeeper-stats-providers/pom.xml
----------------------------------------------------------------------
diff --git a/bookkeeper-stats-providers/pom.xml b/bookkeeper-stats-providers/pom.xml
index 5189579..d6c58b5 100644
--- a/bookkeeper-stats-providers/pom.xml
+++ b/bookkeeper-stats-providers/pom.xml
@@ -25,8 +25,6 @@
     <version>4.5.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
-  <groupId>org.apache.bookkeeper</groupId>
-  <version>4.5.0-SNAPSHOT</version>
   <artifactId>bookkeeper-stats-providers</artifactId>
   <packaging>pom</packaging>
   <name>bookkeeper-stats-providers</name>
@@ -34,6 +32,7 @@
     <module>twitter-science-provider</module>
     <module>twitter-ostrich-provider</module>
     <module>codahale-metrics-provider</module>
+    <module>prometheus-metrics-provider</module>
   </modules>
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/d8635137/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml
----------------------------------------------------------------------
diff --git a/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml b/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml
new file mode 100644
index 0000000..dc13d18
--- /dev/null
+++ b/bookkeeper-stats-providers/prometheus-metrics-provider/pom.xml
@@ -0,0 +1,74 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<project
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+  xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <artifactId>bookkeeper</artifactId>
+    <groupId>org.apache.bookkeeper</groupId>
+    <version>4.5.0-SNAPSHOT</version>
+    <relativePath>../..</relativePath>
+  </parent>
+  <groupId>org.apache.bookkeeper.stats</groupId>
+  <artifactId>prometheus-metrics-provider</artifactId>
+  <name>Stats provider for Prometheus</name>
+
+  <properties>
+    <prometheus.version>0.0.21</prometheus.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.bookkeeper.stats</groupId>
+      <artifactId>bookkeeper-stats-api</artifactId>
+      <version>${project.parent.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>io.prometheus</groupId>
+      <artifactId>simpleclient</artifactId>
+      <version>${prometheus.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>io.prometheus</groupId>
+      <artifactId>simpleclient_hotspot</artifactId>
+      <version>${prometheus.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>io.prometheus</groupId>
+      <artifactId>simpleclient_servlet</artifactId>
+      <version>${prometheus.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-servlet</artifactId>
+      <version>9.4.5.v20170502</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>${guava.version}</version>
+    </dependency>
+
+  </dependencies>
+</project>

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/d8635137/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/PrometheusCounter.java
----------------------------------------------------------------------
diff --git a/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/PrometheusCounter.java
b/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/PrometheusCounter.java
new file mode 100644
index 0000000..5535a3f
--- /dev/null
+++ b/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/PrometheusCounter.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.bookkeeper.stats;
+
+import io.prometheus.client.Collector;
+import io.prometheus.client.CollectorRegistry;
+import io.prometheus.client.Gauge;
+
+public class PrometheusCounter implements Counter {
+
+    private final Gauge gauge;
+
+    public PrometheusCounter(CollectorRegistry registry, String name) {
+        this.gauge = Gauge.build().name(Collector.sanitizeMetricName(name)).help("-").create().register(registry);
+    }
+
+    @Override
+    public void clear() {
+        gauge.clear();
+    }
+
+    @Override
+    public void inc() {
+        gauge.inc();
+    }
+
+    @Override
+    public void dec() {
+        gauge.dec();
+    }
+
+    @Override
+    public void add(long delta) {
+        gauge.inc(delta);
+    }
+
+    @Override
+    public Long get() {
+        return (long) gauge.get();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/d8635137/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/PrometheusMetricsProvider.java
----------------------------------------------------------------------
diff --git a/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/PrometheusMetricsProvider.java
b/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/PrometheusMetricsProvider.java
new file mode 100644
index 0000000..37984de
--- /dev/null
+++ b/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/PrometheusMetricsProvider.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.bookkeeper.stats;
+
+import java.net.InetSocketAddress;
+
+import org.apache.commons.configuration.Configuration;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.prometheus.client.CollectorRegistry;
+import io.prometheus.client.exporter.MetricsServlet;
+import io.prometheus.client.hotspot.GarbageCollectorExports;
+import io.prometheus.client.hotspot.MemoryPoolsExports;
+import io.prometheus.client.hotspot.StandardExports;
+import io.prometheus.client.hotspot.ThreadExports;
+
+public class PrometheusMetricsProvider implements StatsProvider {
+
+    private final CollectorRegistry registry = new CollectorRegistry();
+    private Server server;
+
+    @Override
+    public void start(Configuration conf) {
+        int httpPort = conf.getInt("prometheusStatsHttpPort", 8000);
+        InetSocketAddress httpEndpoint = InetSocketAddress.createUnresolved("0.0.0.0", httpPort);
+        this.server = new Server(httpEndpoint);
+        ServletContextHandler context = new ServletContextHandler();
+        context.setContextPath("/");
+        server.setHandler(context);
+
+        context.addServlet(new ServletHolder(new MetricsServlet(registry)), "/metrics");
+
+        try {
+            server.start();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+
+        log.info("Started Prometheus stats endpoint at {}", httpEndpoint);
+
+        // Include standard JVM stats
+        new StandardExports().register(registry);
+        new MemoryPoolsExports().register(registry);
+        new GarbageCollectorExports().register(registry);
+        new ThreadExports().register(registry);
+    }
+
+    @Override
+    public void stop() {
+        if (server != null) {
+            try {
+                server.stop();
+            } catch (Exception e) {
+                log.warn("Failed to shutdown Jetty server", e);
+            }
+        }
+    }
+
+    @Override
+    public StatsLogger getStatsLogger(String scope) {
+        return new PrometheusStatsLogger(registry, scope);
+    }
+
+    private static final Logger log = LoggerFactory.getLogger(PrometheusMetricsProvider.class);
+}

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/d8635137/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/PrometheusOpStatsLogger.java
----------------------------------------------------------------------
diff --git a/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/PrometheusOpStatsLogger.java
b/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/PrometheusOpStatsLogger.java
new file mode 100644
index 0000000..e10160c
--- /dev/null
+++ b/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/PrometheusOpStatsLogger.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.bookkeeper.stats;
+
+import java.util.concurrent.TimeUnit;
+
+import io.prometheus.client.CollectorRegistry;
+import io.prometheus.client.Summary;
+
+public class PrometheusOpStatsLogger implements OpStatsLogger {
+
+    private final Summary summary;
+    private final Summary.Child success;
+    private final Summary.Child fail;
+
+    public PrometheusOpStatsLogger(CollectorRegistry registry, String name) {
+        this.summary = Summary.build().name(name).help("-") //
+                .quantile(0.50, 0.01) //
+                .quantile(0.75, 0.01) //
+                .quantile(0.95, 0.01) //
+                .quantile(0.99, 0.01) //
+                .quantile(0.999, 0.01) //
+                .quantile(0.9999, 0.01) //
+                .quantile(1.0, 0.01) //
+                .maxAgeSeconds(60) //
+                .labelNames("success") //
+                .create().register(registry);
+
+        this.success = summary.labels("true");
+        this.fail = summary.labels("false");
+    }
+
+    @Override
+    public void registerSuccessfulEvent(long eventLatency, TimeUnit unit) {
+        // Collect latency in millis, truncating anything below micros
+        success.observe(unit.toMicros(eventLatency) / 1000.0);
+    }
+
+    @Override
+    public void registerFailedEvent(long eventLatency, TimeUnit unit) {
+        fail.observe(unit.toMicros(eventLatency) / 1000.0);
+    }
+
+    @Override
+    public void registerSuccessfulValue(long value) {
+        success.observe(value);
+    }
+
+    @Override
+    public void registerFailedValue(long value) {
+        fail.observe(value);
+    }
+
+    @Override
+    public OpStatsData toOpStatsData() {
+        // Not relevant as we don't use JMX here
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void clear() {
+        summary.clear();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/d8635137/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/PrometheusStatsLogger.java
----------------------------------------------------------------------
diff --git a/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/PrometheusStatsLogger.java
b/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/PrometheusStatsLogger.java
new file mode 100644
index 0000000..ab15639
--- /dev/null
+++ b/bookkeeper-stats-providers/prometheus-metrics-provider/src/main/java/org/apache/bookkeeper/stats/PrometheusStatsLogger.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.bookkeeper.stats;
+
+import com.google.common.base.Joiner;
+
+import io.prometheus.client.Collector;
+import io.prometheus.client.CollectorRegistry;
+
+public class PrometheusStatsLogger implements StatsLogger {
+
+    private final CollectorRegistry registry;
+    private final String scope;
+
+    PrometheusStatsLogger(CollectorRegistry registry, String scope) {
+        this.registry = registry;
+        this.scope = scope;
+    }
+
+    @Override
+    public OpStatsLogger getOpStatsLogger(String name) {
+        return new PrometheusOpStatsLogger(registry, completeName(name));
+    }
+
+    @Override
+    public Counter getCounter(String name) {
+        return new PrometheusCounter(registry, completeName(name));
+    }
+
+    @Override
+    public <T extends Number> void registerGauge(String name, Gauge<T> gauge)
{
+        io.prometheus.client.Gauge.build().name(completeName(name)).help("-").create()
+                .setChild(new io.prometheus.client.Gauge.Child() {
+                    @Override
+                    public double get() {
+                        Number value = null;
+                        try {
+                            value = gauge.getSample();
+                        } catch (Exception e) {
+                            // no-op
+                        }
+
+                        if (value == null) {
+                            value = gauge.getDefaultValue();
+                        }
+                        return value.doubleValue();
+                    }
+                }).register(registry);
+    }
+
+    @Override
+    public StatsLogger scope(String name) {
+        return new PrometheusStatsLogger(registry, completeName(name));
+    }
+
+    private String completeName(String name) {
+        String completeName = scope.isEmpty() ? name : Joiner.on('_').join(scope, name);
+        return Collector.sanitizeMetricName(completeName);
+    }
+}

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/d8635137/bookkeeper-stats/pom.xml
----------------------------------------------------------------------
diff --git a/bookkeeper-stats/pom.xml b/bookkeeper-stats/pom.xml
index 073eeef..a3389e7 100644
--- a/bookkeeper-stats/pom.xml
+++ b/bookkeeper-stats/pom.xml
@@ -46,12 +46,7 @@
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
-      <version>1.6.4</version>
-    </dependency>
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-log4j12</artifactId>
-      <version>1.6.4</version>
+      <version>${slf4j.version}</version>
     </dependency>
   </dependencies>
 </project>

http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/d8635137/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a892cef..76b4d24 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,6 +45,7 @@
     <guava.version>13.0.1</guava.version>
     <netty.version>4.1.10.Final</netty.version>
     <zookeeper.version>3.5.1-alpha</zookeeper.version>
+    <slf4j.version>1.7.25</slf4j.version>
   </properties>
   <url>http://zookeeper.apache.org/bookkeeper</url>
   <build>


Mime
View raw message