zookeeper-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From si...@apache.org
Subject svn commit: r1540366 - in /zookeeper/bookkeeper/trunk: ./ bookkeeper-stats-providers/ bookkeeper-stats-providers/codahale-metrics-provider/ bookkeeper-stats-providers/codahale-metrics-provider/src/ bookkeeper-stats-providers/codahale-metrics-provider/s...
Date Sat, 09 Nov 2013 18:49:55 GMT
Author: sijie
Date: Sat Nov  9 18:49:55 2013
New Revision: 1540366

URL: http://svn.apache.org/r1540366
Log:
BOOKKEEPER-699: Codahale metrics implementation of stats API (ivank via sijie)

Added:
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsProvider.java
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleOpStatsLogger.java
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleStatsLogger.java
Modified:
    zookeeper/bookkeeper/trunk/CHANGES.txt
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/pom.xml

Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1540366&r1=1540365&r2=1540366&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Sat Nov  9 18:49:55 2013
@@ -190,6 +190,8 @@ Trunk (unreleased changes)
 
       BOOKKEEPER-698: Bookie client closure is not considering timeoutExecutor (rakesh via
sijie)
 
+      BOOKKEEPER-699: Codahale metrics implementation of stats API (ivank via sijie)
+
     NEW FEATURE:
 
       BOOKKEEPER-562: Ability to tell if a ledger is closed or not (fpj)

Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml?rev=1540366&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml
(added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/pom.xml
Sat Nov  9 18:49:55 2013
@@ -0,0 +1,62 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <artifactId>bookkeeper</artifactId>
+    <groupId>org.apache.bookkeeper</groupId>
+    <version>4.3.0-SNAPSHOT</version>
+    <relativePath>../..</relativePath>
+  </parent>
+  <groupId>org.apache.bookkeeper.stats</groupId>
+  <artifactId>codahale-metrics-provider</artifactId>
+  <name>Stats provider for codahale metrics</name>
+  <url>http://maven.apache.org</url>
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <project.libdir>${basedir}/lib</project.libdir>
+  </properties>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.bookkeeper.stats</groupId>
+      <artifactId>bookkeeper-stats-api</artifactId>
+      <version>${project.parent.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.codahale.metrics</groupId>
+      <artifactId>metrics-core</artifactId>
+      <version>3.0.1</version>
+    </dependency>
+    <dependency>
+      <groupId>com.codahale.metrics</groupId>
+      <artifactId>metrics-jvm</artifactId>
+      <version>3.0.1</version>
+    </dependency>
+    <dependency>
+      <groupId>com.codahale.metrics</groupId>
+      <artifactId>metrics-graphite</artifactId>
+      <version>3.0.1</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>13.0.1</version>
+    </dependency>
+  </dependencies>
+</project>

Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsProvider.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsProvider.java?rev=1540366&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsProvider.java
(added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsProvider.java
Sat Nov  9 18:49:55 2013
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.bookkeeper.stats;
+
+import java.net.InetSocketAddress;
+
+import com.codahale.metrics.CsvReporter;
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.ScheduledReporter;
+import com.codahale.metrics.Slf4jReporter;
+import com.google.common.base.Strings;
+import com.google.common.net.HostAndPort;
+import static com.codahale.metrics.MetricRegistry.name;
+import com.codahale.metrics.graphite.Graphite;
+import com.codahale.metrics.graphite.GraphiteReporter;
+
+import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
+import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
+
+import org.apache.bookkeeper.stats.StatsLogger;
+import org.apache.bookkeeper.stats.StatsProvider;
+import org.apache.commons.configuration.Configuration;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+
+import java.io.File;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CodahaleMetricsProvider implements StatsProvider {
+
+    static final Logger LOG = LoggerFactory.getLogger(CodahaleMetricsProvider.class);
+
+    MetricRegistry metrics = null;
+    List<ScheduledReporter> reporters = new ArrayList<ScheduledReporter>();
+
+    synchronized void initIfNecessary() {
+        if (metrics == null) {
+            metrics = new MetricRegistry();
+            metrics.registerAll(new MemoryUsageGaugeSet());
+            metrics.registerAll(new GarbageCollectorMetricSet());
+        }
+    }
+
+    @Override
+    public void start(Configuration conf) {
+        initIfNecessary();
+
+        int metricsOutputFrequency = conf.getInt("codahaleStatsOutputFrequencySeconds", 60);
+        String prefix = conf.getString("codahaleStatsPrefix", "");
+        String graphiteHost = conf.getString("codahaleStatsGraphiteEndpoint");
+        String csvDir = conf.getString("codahaleStatsCSVEndpoint");
+        String slf4jCat = conf.getString("codahaleStatsSlf4jEndpoint");
+
+        if (!Strings.isNullOrEmpty(graphiteHost)) {
+            LOG.info("Configuring stats with graphite");
+            HostAndPort addr = HostAndPort.fromString(graphiteHost);
+            final Graphite graphite = new Graphite(
+                    new InetSocketAddress(addr.getHostText(), addr.getPort()));
+            reporters.add(GraphiteReporter.forRegistry(metrics)
+                          .prefixedWith(prefix)
+                          .convertRatesTo(TimeUnit.SECONDS)
+                          .convertDurationsTo(TimeUnit.MILLISECONDS)
+                          .filter(MetricFilter.ALL)
+                          .build(graphite));
+        }
+        if (!Strings.isNullOrEmpty(csvDir)) {
+            // NOTE: 1/ metrics output files are exclusive to a given process
+            // 2/ the output directory must exist
+            // 3/ if output files already exist they are not overwritten and there is no
metrics output
+            File outdir;
+            if (Strings.isNullOrEmpty(prefix)) {
+                outdir = new File(csvDir, prefix);
+            } else {
+                outdir = new File(csvDir);
+            }
+            LOG.info("Configuring stats with csv output to directory [{}]", outdir.getAbsolutePath());
+            reporters.add(CsvReporter.forRegistry(metrics)
+                          .convertRatesTo(TimeUnit.SECONDS)
+                          .convertDurationsTo(TimeUnit.MILLISECONDS)
+                          .build(outdir));
+        }
+        if (!Strings.isNullOrEmpty(slf4jCat)) {
+            LOG.info("Configuring stats with slf4j");
+            reporters.add(Slf4jReporter.forRegistry(metrics)
+                          .outputTo(LoggerFactory.getLogger(slf4jCat))
+                          .convertRatesTo(TimeUnit.SECONDS)
+                          .convertDurationsTo(TimeUnit.MILLISECONDS)
+                          .build());
+        }
+        for (ScheduledReporter r : reporters) {
+            r.start(metricsOutputFrequency, TimeUnit.SECONDS);
+        }
+    }
+
+    @Override
+    public void stop() {
+        for (ScheduledReporter r : reporters) {
+            r.report();
+            r.stop();
+        }
+    }
+
+    @Override
+    public StatsLogger getStatsLogger(String name) {
+        initIfNecessary();
+        return new CodahaleStatsLogger(metrics, name);
+    }
+}

Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleOpStatsLogger.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleOpStatsLogger.java?rev=1540366&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleOpStatsLogger.java
(added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleOpStatsLogger.java
Sat Nov  9 18:49:55 2013
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.bookkeeper.stats;
+
+import org.apache.bookkeeper.stats.OpStatsData;
+import org.apache.bookkeeper.stats.OpStatsLogger;
+
+import com.codahale.metrics.Timer;
+import com.codahale.metrics.Snapshot;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+class CodahaleOpStatsLogger implements OpStatsLogger {
+    final Timer success;
+    final Timer fail;
+
+    CodahaleOpStatsLogger(Timer success, Timer fail) {
+        this.success = success;
+        this.fail = fail;
+    }
+
+    // OpStatsLogger functions
+    public void registerFailedEvent(long eventLatencyMillis) {
+        fail.update(eventLatencyMillis, TimeUnit.MILLISECONDS);
+    }
+
+    public void registerSuccessfulEvent(long eventLatencyMillis) {
+        success.update(eventLatencyMillis, TimeUnit.MILLISECONDS);
+    }
+
+    public synchronized void clear() {
+        // can't clear a timer
+    }
+
+    /**
+     * This function should go away soon (hopefully).
+     */
+    public synchronized OpStatsData toOpStatsData() {
+        long numFailed = fail.getCount();
+        long numSuccess = success.getCount();
+        Snapshot s = success.getSnapshot();
+        double avgLatencyMillis = s.getMean();
+
+        double[] default_percentiles = {10, 50, 90, 99, 99.9, 99.99};
+        long[] latenciesMillis = new long[default_percentiles.length];
+        Arrays.fill(latenciesMillis, Long.MAX_VALUE);
+        for (int i = 0; i < default_percentiles.length; i++) {
+            latenciesMillis[i] = (long)s.getValue(default_percentiles[i]);
+        }
+        return new OpStatsData(numSuccess, numFailed, avgLatencyMillis, latenciesMillis);
+    }
+}

Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleStatsLogger.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleStatsLogger.java?rev=1540366&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleStatsLogger.java
(added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/codahale-metrics-provider/src/main/java/org/apache/bookkeeper/stats/CodahaleStatsLogger.java
Sat Nov  9 18:49:55 2013
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.bookkeeper.stats;
+
+import com.codahale.metrics.Timer;
+import com.codahale.metrics.MetricRegistry;
+import static com.codahale.metrics.MetricRegistry.name;
+
+import org.apache.bookkeeper.stats.Counter;
+import org.apache.bookkeeper.stats.Gauge;
+import org.apache.bookkeeper.stats.OpStatsLogger;
+import org.apache.bookkeeper.stats.StatsLogger;
+
+public class CodahaleStatsLogger implements StatsLogger {
+    protected final String basename;
+    final MetricRegistry metrics;
+
+    CodahaleStatsLogger(MetricRegistry metrics, String basename) {
+        this.metrics = metrics;
+        this.basename = basename;
+    }
+
+    @Override
+    public OpStatsLogger getOpStatsLogger(String statName) {
+        Timer success = metrics.timer(name(basename, statName));
+        Timer failure = metrics.timer(name(basename, statName+"-fail"));
+        return new CodahaleOpStatsLogger(success, failure);
+    }
+
+    @Override
+    public Counter getCounter(String statName) {
+        final com.codahale.metrics.Counter c = metrics.counter(name(basename, statName));
+        return new Counter() {
+            @Override
+            public synchronized void clear() {
+                long cur = c.getCount();
+                c.dec(cur);
+            }
+
+            @Override
+            public Long get() {
+                return c.getCount();
+            }
+
+            @Override
+            public void inc() {
+                c.inc();
+            }
+
+            @Override
+            public void dec() {
+                c.dec();
+            }
+
+            @Override
+            public void add(long delta) {
+                c.inc(delta);
+            }
+        };
+    }
+
+    @Override
+    public <T extends Number> void registerGauge(final String statName, final Gauge<T>
gauge) {
+        String metricName = name(basename, statName);
+        metrics.remove(metricName);
+
+        metrics.register(metricName, new com.codahale.metrics.Gauge<T>() {
+                @Override
+                public T getValue() {
+                    return gauge.getSample();
+                }
+            });
+    }
+
+    @Override
+    public StatsLogger scope(String scope) {
+        String scopeName;
+        if (0 == basename.length()) {
+            scopeName = scope;
+        } else {
+            scopeName = name(basename, scope);
+        }
+        return new CodahaleStatsLogger(metrics, scopeName);
+    }
+}

Modified: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/pom.xml
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/pom.xml?rev=1540366&r1=1540365&r2=1540366&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/pom.xml (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/pom.xml Sat Nov  9 18:49:55 2013
@@ -33,6 +33,7 @@
   <modules>
     <module>twitter-science-provider</module>
     <module>twitter-ostrich-provider</module>
+    <module>codahale-metrics-provider</module>
   </modules>
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>



Mime
View raw message