zookeeper-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From iv...@apache.org
Subject svn commit: r1536635 - in /zookeeper/bookkeeper/trunk: ./ bookkeeper-stats-providers/ bookkeeper-stats-providers/twitter-ostrich-provider/ bookkeeper-stats-providers/twitter-ostrich-provider/src/ bookkeeper-stats-providers/twitter-ostrich-provider/src/...
Date Tue, 29 Oct 2013 09:58:03 GMT
Author: ivank
Date: Tue Oct 29 09:58:02 2013
New Revision: 1536635

URL: http://svn.apache.org/r1536635
Log:
BOOKKEEPER-615: Twitter stats implementation of stats interface (sijie via ivank)

Added:
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/pom.xml
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/pom.xml
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/CounterImpl.java
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OpStatsLoggerImpl.java
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OstrichProvider.java
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OstrichStatsLoggerImpl.java
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/pom.xml
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/CounterImpl.java
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/HTTPStatsExporter.java
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/OpStatsLoggerImpl.java
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/TwitterStatsLoggerImpl.java
    zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/TwitterStatsProvider.java
Modified:
    zookeeper/bookkeeper/trunk/CHANGES.txt

Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1536635&r1=1536634&r2=1536635&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Tue Oct 29 09:58:02 2013
@@ -52,6 +52,8 @@ Trunk (unreleased changes)
 
       BOOKKEEPER-614: Generic stats interface, which multiple providers can be plugged into
(sijie & ivank via ivank)
 
+      BOOKKEEPER-615: Twitter stats implementation of stats interface (sijie via ivank)
+
       bookkeeper-server:
 
         BOOKKEEPER-567: ReadOnlyBookieTest hangs on shutdown (sijie via ivank)

Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/pom.xml
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/pom.xml?rev=1536635&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/pom.xml (added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/pom.xml Tue Oct 29 09:58:02 2013
@@ -0,0 +1,41 @@
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+                      http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <artifactId>bookkeeper</artifactId>
+    <groupId>org.apache.bookkeeper</groupId>
+    <version>4.3.0-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.apache.bookkeeper</groupId>
+  <version>4.3.0-SNAPSHOT</version>
+  <artifactId>bookkeeper-stats-providers</artifactId>
+  <packaging>pom</packaging>
+  <name>bookkeeper-stats-providers</name>
+  <modules>
+    <module>twitter-science-provider</module>
+    <module>twitter-ostrich-provider</module>
+  </modules>
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+  </properties>
+</project>

Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/pom.xml
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/pom.xml?rev=1536635&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/pom.xml
(added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/pom.xml
Tue Oct 29 09:58:02 2013
@@ -0,0 +1,55 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <artifactId>bookkeeper</artifactId>
+    <groupId>org.apache.bookkeeper</groupId>
+    <version>4.3.0-SNAPSHOT</version>
+    <relativePath>../..</relativePath>
+  </parent>
+  <groupId>org.apache.bookkeeper.stats</groupId>
+  <artifactId>twitter-ostrich-provider</artifactId>
+  <name>Stats provider for twitter-ostrich package</name>
+  <url>http://maven.apache.org</url>
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <project.libdir>${basedir}/lib</project.libdir>
+  </properties>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.bookkeeper.stats</groupId>
+      <artifactId>bookkeeper-stats-api</artifactId>
+      <version>${project.parent.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.twitter</groupId>
+      <artifactId>ostrich_2.9.2</artifactId>
+      <version>9.1.3</version>
+    </dependency>
+  </dependencies>
+  <repositories>
+    <repository>
+      <id>twitter</id>
+      <name>Twitter repo</name>
+      <layout>default</layout>
+      <url>http://maven.twttr.com</url>
+    </repository>
+  </repositories>
+</project> 

Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/CounterImpl.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/CounterImpl.java?rev=1536635&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/CounterImpl.java
(added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/CounterImpl.java
Tue Oct 29 09:58:02 2013
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.bookkeeper.stats.twitter.ostrich;
+
+import org.apache.bookkeeper.stats.Counter;
+
+class CounterImpl implements Counter {
+
+    private final com.twitter.ostrich.stats.Counter ostrichCounter;
+
+    CounterImpl(com.twitter.ostrich.stats.Counter ostrichCounter) {
+        this.ostrichCounter = ostrichCounter;
+    }
+
+    @Override
+    public void clear() {
+        this.ostrichCounter.reset();
+    }
+
+    @Override
+    public void inc() {
+        this.ostrichCounter.incr();
+    }
+
+    @Override
+    public void dec() {
+        this.ostrichCounter.incr(-1);
+    }
+
+    @Override
+    public void add(long delta) {
+        this.ostrichCounter.incr((int)delta);
+    }
+
+    @Override
+    public Long get() {
+        return this.ostrichCounter.apply();
+    }
+}

Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OpStatsLoggerImpl.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OpStatsLoggerImpl.java?rev=1536635&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OpStatsLoggerImpl.java
(added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OpStatsLoggerImpl.java
Tue Oct 29 09:58:02 2013
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.bookkeeper.stats.twitter.ostrich;
+
+import org.apache.bookkeeper.stats.OpStatsData;
+import org.apache.bookkeeper.stats.OpStatsLogger;
+
+class OpStatsLoggerImpl implements OpStatsLogger {
+
+    static final double[] PERCENTILES = new double[] {
+            0.1, 0.5, 0.9, 0.99, 0.999, 0.9999
+    };
+
+    private final String scope;
+    private final com.twitter.ostrich.stats.Counter successCounter;
+    private final com.twitter.ostrich.stats.Counter failureCounter;
+    private final com.twitter.ostrich.stats.Metric successMetric;
+    private final com.twitter.ostrich.stats.Metric failureMetric;
+
+    OpStatsLoggerImpl(String scope, com.twitter.ostrich.stats.StatsProvider statsProvider)
{
+        this.scope = scope;
+        successCounter = statsProvider.getCounter(statName("requests/success"));
+        failureCounter = statsProvider.getCounter(statName("requests/failure"));
+        successMetric = statsProvider.getMetric(statName("latency/success"));
+        failureMetric = statsProvider.getMetric(statName("latency/failure"));
+    }
+
+    private String statName(String statName) {
+        return String.format("%s/%s", scope, statName);
+    }
+
+    @Override
+    public void registerFailedEvent(long eventLatencyMillis) {
+        failureMetric.add((int)eventLatencyMillis);
+        failureCounter.incr();
+    }
+
+    @Override
+    public void registerSuccessfulEvent(long eventLatencyMillis) {
+        successMetric.add((int)eventLatencyMillis);
+        successCounter.incr();
+    }
+
+    @Override
+    public OpStatsData toOpStatsData() {
+        long numSuccess = successCounter.apply();
+        long numFailures = failureCounter.apply();
+        com.twitter.ostrich.stats.Distribution distribution = successMetric.apply();
+        com.twitter.ostrich.stats.Histogram histogram = distribution.histogram();
+        double avgLatency = distribution.average();
+        long[] percentiles = new long[PERCENTILES.length];
+        int i = 0;
+        for (double percentile : PERCENTILES) {
+            percentiles[i] = histogram.getPercentile(percentile);
+            ++i;
+        }
+        return new OpStatsData(numSuccess, numFailures, avgLatency, percentiles);
+    }
+
+    @Override
+    public void clear() {
+        successCounter.reset();
+        failureCounter.reset();
+        successMetric.clear();
+        failureMetric.clear();
+    }
+}

Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OstrichProvider.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OstrichProvider.java?rev=1536635&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OstrichProvider.java
(added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OstrichProvider.java
Tue Oct 29 09:58:02 2013
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.bookkeeper.stats.twitter.ostrich;
+
+import com.twitter.ostrich.admin.CustomHttpHandler;
+import com.twitter.ostrich.admin.RuntimeEnvironment;
+import com.twitter.util.Duration;
+import org.apache.bookkeeper.stats.StatsLogger;
+import org.apache.bookkeeper.stats.StatsProvider;
+import org.apache.commons.configuration.Configuration;
+import scala.Some;
+import scala.collection.immutable.$colon$colon;
+import scala.collection.immutable.List;
+import scala.collection.immutable.List$;
+import scala.collection.immutable.Map;
+import scala.collection.immutable.Map$;
+
+import java.util.concurrent.TimeUnit;
+
+public class OstrichProvider implements StatsProvider {
+
+    protected final static String STATS_EXPORT = "statsExport";
+    protected final static String STATS_HTTP_PORT = "statsHttpPort";
+
+    private com.twitter.ostrich.admin.AdminHttpService statsExporter = null;
+
+    private static <T> List<T> list(T ... ts) {
+        List<T> result = List$.MODULE$.empty();
+        for (int i = ts.length; i > 0; i--) {
+            result = new $colon$colon<T>(ts[i-1], result);
+        }
+        return result;
+    }
+
+    private static <K, V> Map<K, V> emptyMap() {
+        Map<K, V> result = Map$.MODULE$.empty();
+        return result;
+    }
+
+    @Override
+    public void start(Configuration conf) {
+        if (conf.getBoolean(STATS_EXPORT, false)) {
+            statsExporter = new com.twitter.ostrich.admin.AdminServiceFactory(
+                    conf.getInt(STATS_HTTP_PORT, 9002), 20, null, Some.apply(""), null,
+                    OstrichProvider.<String, CustomHttpHandler>emptyMap(), list(Duration.apply(1,
TimeUnit.MINUTES))
+            ).apply(RuntimeEnvironment.apply(this, new String[0]));
+        }
+        if (null != statsExporter) {
+            statsExporter.start();
+        }
+    }
+
+    @Override
+    public void stop() {
+        if (null != statsExporter) {
+            statsExporter.shutdown();
+        }
+    }
+
+    @Override
+    public StatsLogger getStatsLogger(String scope) {
+        return new OstrichStatsLoggerImpl(scope, com.twitter.ostrich.stats.Stats.get(""));
+    }
+}

Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OstrichStatsLoggerImpl.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OstrichStatsLoggerImpl.java?rev=1536635&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OstrichStatsLoggerImpl.java
(added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-ostrich-provider/src/main/java/org/apache/bookkeeper/stats/twitter/ostrich/OstrichStatsLoggerImpl.java
Tue Oct 29 09:58:02 2013
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.bookkeeper.stats.twitter.ostrich;
+
+import org.apache.bookkeeper.stats.Counter;
+import org.apache.bookkeeper.stats.Gauge;
+import org.apache.bookkeeper.stats.OpStatsLogger;
+import org.apache.bookkeeper.stats.StatsLogger;
+import scala.Function0;
+import scala.runtime.AbstractFunction0;
+
+/**
+ * Implementation of ostrich logger.
+ */
+class OstrichStatsLoggerImpl implements StatsLogger {
+
+    protected final String scope;
+    protected final com.twitter.ostrich.stats.StatsProvider ostrichProvider;
+
+    OstrichStatsLoggerImpl(String scope, com.twitter.ostrich.stats.StatsProvider ostrichProvider)
{
+        this.scope = scope;
+        this.ostrichProvider = ostrichProvider;
+    }
+
+    @Override
+    public OpStatsLogger getOpStatsLogger(String statName) {
+        return new OpStatsLoggerImpl(getStatName(statName), ostrichProvider);
+    }
+
+    @Override
+    public Counter getCounter(String statName) {
+        return new CounterImpl(ostrichProvider.getCounter(getStatName(statName)));
+    }
+
+    @Override
+    public <T extends Number> void registerGauge(final String statName, final Gauge<T>
gauge) {
+        Function0<Object> gaugeFunc = new AbstractFunction0<Object>() {
+            @Override
+            public Object apply() {
+                return gauge.getSample().doubleValue();
+            }
+        };
+        ostrichProvider.addGauge(getStatName(statName), gaugeFunc);
+    }
+
+    private String getStatName(String statName) {
+        return String.format("%s/%s", scope, statName);
+    }
+
+    @Override
+    public StatsLogger scope(String scope) {
+        return new OstrichStatsLoggerImpl(getStatName(scope), ostrichProvider);
+    }
+
+}

Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/pom.xml
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/pom.xml?rev=1536635&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/pom.xml
(added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/pom.xml
Tue Oct 29 09:58:02 2013
@@ -0,0 +1,89 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <artifactId>bookkeeper</artifactId>
+    <groupId>org.apache.bookkeeper</groupId>
+    <version>4.3.0-SNAPSHOT</version>
+    <relativePath>../..</relativePath>
+  </parent>
+  <groupId>org.apache.bookkeeper.stats</groupId>
+  <artifactId>twitter-science-provider</artifactId>
+  <name>Stats provider for twitter-stats package</name>
+  <url>http://maven.apache.org</url>
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <project.libdir>${basedir}/lib</project.libdir>
+  </properties>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.bookkeeper.stats</groupId>
+      <artifactId>bookkeeper-stats-api</artifactId>
+      <version>${project.parent.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.twitter.common</groupId>
+      <artifactId>stats</artifactId>
+      <version>0.0.64</version>
+    </dependency>
+    <dependency>
+      <groupId>com.twitter.common</groupId>
+      <artifactId>net-http-handlers</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>javax.servlet</groupId>
+          <artifactId>servlet-api</artifactId>
+        </exclusion>
+      </exclusions>
+      <version>0.0.39</version>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.twitter.common</groupId>
+      <artifactId>stats-time-series</artifactId>
+      <version>0.0.36</version>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.twitter.common</groupId>
+      <artifactId>stats-jvm</artifactId>
+      <version>0.0.33</version>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-server</artifactId>
+      <version>8.1.4.v20120524</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-servlet</artifactId>
+      <version>8.1.4.v20120524</version>
+    </dependency>
+  </dependencies>
+  <repositories>
+    <repository>
+      <id>twitter</id>
+      <name>Twitter repo</name>
+      <layout>default</layout>
+      <url>http://maven.twttr.com</url>
+    </repository>
+  </repositories>
+</project> 

Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/CounterImpl.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/CounterImpl.java?rev=1536635&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/CounterImpl.java
(added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/CounterImpl.java
Tue Oct 29 09:58:02 2013
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.bookkeeper.stats.twitter.science;
+
+import com.twitter.common.stats.Rate;
+import com.twitter.common.stats.Stats;
+import org.apache.bookkeeper.stats.Counter;
+
+import java.util.concurrent.atomic.AtomicLong;
+/**
+ * This will export the value and the rate (per sec) to {@link org.apache.bookkeeper.stats.Stats}
+ */
+public class CounterImpl implements Counter {
+    // The name used to export this stat
+    private String name;
+    private AtomicLong value;
+
+    public CounterImpl(String name) {
+        this.name = name;
+        value = new AtomicLong(0);
+        setUpStatsExport();
+    }
+
+    @Override
+    public synchronized void clear() {
+        value.getAndSet(0);
+    }
+
+    @Override
+    public Long get() {
+        return value.get();
+    }
+
+    private void setUpStatsExport() {
+        // Export the value.
+        Stats.export(name, value);
+        // Export the rate of this value.
+        Stats.export(Rate.of(name + "_per_sec", value).build());
+    }
+
+    @Override
+    public void inc() {
+        value.incrementAndGet();
+    }
+
+    @Override
+    public void dec() {
+        value.decrementAndGet();
+    }
+
+    @Override
+    public void add(long delta) {
+        value.addAndGet(delta);
+    }
+}

Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/HTTPStatsExporter.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/HTTPStatsExporter.java?rev=1536635&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/HTTPStatsExporter.java
(added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/HTTPStatsExporter.java
Tue Oct 29 09:58:02 2013
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bookkeeper.stats.twitter.science;
+
+import com.twitter.common.application.ShutdownRegistry;
+import com.twitter.common.base.Supplier;
+import com.twitter.common.net.http.handlers.VarsHandler;
+import com.twitter.common.net.http.handlers.VarsJsonHandler;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.stats.*;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+
+/**
+ * Starts a jetty server on a configurable port and the samplers to export stats.
+ */
+public class HTTPStatsExporter {
+    final Server jettyServer;
+    final ShutdownRegistry.ShutdownRegistryImpl shutDownRegistry;
+    final int port;
+
+    public HTTPStatsExporter(int port) {
+        // Create the ShutdownRegistry needed for our sampler
+        this.shutDownRegistry = new ShutdownRegistry.ShutdownRegistryImpl();
+        this.port = port;
+        this.jettyServer = new Server(port);
+    }
+
+    public void start() throws Exception {
+        // Start the sampler. Sample every 1 second and retain for 1 hour
+        // TODO(Aniruddha): Make this configurable if needed.
+        TimeSeriesRepository sampler = new TimeSeriesRepositoryImpl(Stats.STAT_REGISTRY,
+                Amount.of(1L, Time.SECONDS), Amount.of(1L, Time.HOURS));
+        sampler.start(this.shutDownRegistry);
+        // Export JVM stats
+        JvmStats.export();
+        // Configure handlers
+        Supplier<Iterable<Stat<?>>> supplier = new Supplier<Iterable<Stat<?>>>()
{
+            @Override
+            public Iterable<Stat<?>> get() {
+                return Stats.getVariables();
+            }
+        };
+
+        // Start jetty.
+        ServletContextHandler context = new ServletContextHandler();
+        context.setContextPath("/");
+        this.jettyServer.setHandler(context);
+        context.addServlet(new ServletHolder(new VarsHandler(supplier)), "/vars");
+        context.addServlet(new ServletHolder(new VarsJsonHandler(supplier)), "/vars.json");
+        this.jettyServer.start();
+    }
+
+    public void stop() throws Exception {
+        this.jettyServer.stop();
+        if (this.shutDownRegistry != null) {
+            this.shutDownRegistry.execute();
+        }
+    }
+}

Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/OpStatsLoggerImpl.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/OpStatsLoggerImpl.java?rev=1536635&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/OpStatsLoggerImpl.java
(added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/OpStatsLoggerImpl.java
Tue Oct 29 09:58:02 2013
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.bookkeeper.stats.twitter.science;
+
+import com.twitter.common.stats.*;
+import org.apache.bookkeeper.stats.OpStatsData;
+import org.apache.bookkeeper.stats.OpStatsLogger;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * An implementation of the OpStatsLogger interface that handles per operation type stats.
+ * Internals use twitter.common.stats for exporting metrics.
+ */
+public class OpStatsLoggerImpl implements OpStatsLogger {
+    private final RequestStats events;
+
+    public OpStatsLoggerImpl(String name) {
+        this.events = new RequestStats(name);
+    }
+
+    // OpStatsLogger functions
+    public void registerFailedEvent(long eventLatencyMillis) {
+        this.events.incErrors(TimeUnit.MILLISECONDS.toMicros(eventLatencyMillis));
+    }
+
+    public void registerSuccessfulEvent(long eventLatencyMillis) {
+        this.events.requestComplete(TimeUnit.MILLISECONDS.toMicros(eventLatencyMillis));
+    }
+
+    public synchronized void clear() {
+        //TODO(Aniruddha): Figure out how to clear RequestStats. Till then this is a no-op
+    }
+
+    /**
+     * This function should go away soon (hopefully).
+     */
+    public synchronized OpStatsData toOpStatsData() {
+        long numFailed = this.events.getErrorCount();
+        long numSuccess = this.events.getSlidingStats().getEventCounter().get() - numFailed;
+        double avgLatencyMillis = this.events.getSlidingStats().getPerEventLatency().read()
/ 1000.0;
+        double[] default_percentiles = {10, 50, 90, 99, 99.9, 99.99};
+        long[] latenciesMillis = new long[default_percentiles.length];
+        Arrays.fill(latenciesMillis, Long.MAX_VALUE);
+        Map<Double, ? extends Stat> realPercentileLatencies =
+                this.events.getPercentile().getPercentiles();
+        for (int i = 0; i < default_percentiles.length; i++) {
+            if (realPercentileLatencies.containsKey(default_percentiles[i])) {
+                @SuppressWarnings("unchecked")
+                Stat<Double> latency = realPercentileLatencies.get(default_percentiles[i]);
+                latenciesMillis[i] = TimeUnit.MICROSECONDS.toMillis(latency.read().longValue());
+            }
+        }
+        return new OpStatsData(numSuccess, numFailed, avgLatencyMillis, latenciesMillis);
+    }
+}

Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/TwitterStatsLoggerImpl.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/TwitterStatsLoggerImpl.java?rev=1536635&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/TwitterStatsLoggerImpl.java
(added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/TwitterStatsLoggerImpl.java
Tue Oct 29 09:58:02 2013
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.bookkeeper.stats.twitter.science;
+
+import com.twitter.common.stats.SampledStat;
+import com.twitter.common.stats.Stats;
+import org.apache.bookkeeper.stats.Counter;
+import org.apache.bookkeeper.stats.Gauge;
+import org.apache.bookkeeper.stats.OpStatsLogger;
+import org.apache.bookkeeper.stats.StatsLogger;
+
+/**
+ * Implementation of twitter-stats logger.
+ */
+public class TwitterStatsLoggerImpl implements StatsLogger {
+
+    protected final String name;
+
+    public TwitterStatsLoggerImpl(String name) {
+        this.name = name;
+    }
+
+    @Override
+    public OpStatsLogger getOpStatsLogger(String statName) {
+        return new OpStatsLoggerImpl(getStatName(statName));
+    }
+
+    @Override
+    public Counter getCounter(String statName) {
+        return new CounterImpl(getStatName(statName));
+    }
+
+    @Override
+    public <T extends Number> void registerGauge(final String statName, final Gauge<T>
gauge) {
+        Stats.export(new SampledStat<Number>(getStatName(statName), gauge.getDefaultValue())
{
+            @Override
+            public T doSample() {
+                return gauge.getSample();
+            }
+        });
+    }
+
+    private String getStatName(String statName) {
+        return (name + "_" + statName).toLowerCase();
+    }
+
+    @Override
+    public StatsLogger scope(String scope) {
+        String scopeName;
+        if (0 == name.length()) {
+            scopeName = scope;
+        } else {
+            scopeName = name + "_" + scope;
+        }
+        return new TwitterStatsLoggerImpl(scopeName);
+    }
+
+}

Added: zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/TwitterStatsProvider.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/TwitterStatsProvider.java?rev=1536635&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/TwitterStatsProvider.java
(added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-stats-providers/twitter-science-provider/src/main/java/org/apache/bookkeeper/stats/twitter/science/TwitterStatsProvider.java
Tue Oct 29 09:58:02 2013
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.bookkeeper.stats.twitter.science;
+
+import org.apache.bookkeeper.stats.StatsLogger;
+import org.apache.bookkeeper.stats.StatsProvider;
+import org.apache.commons.configuration.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TwitterStatsProvider implements StatsProvider {
+
+    static final Logger LOG = LoggerFactory.getLogger(TwitterStatsProvider.class);
+
+    protected final static String STATS_EXPORT = "statsExport";
+    protected final static String STATS_HTTP_PORT = "statsHttpPort";
+
+    private HTTPStatsExporter statsExporter = null;
+
+    @Override
+    public void start(Configuration conf) {
+        if (conf.getBoolean(STATS_EXPORT, false)) {
+            statsExporter = new HTTPStatsExporter(conf.getInt(STATS_HTTP_PORT, 9002));
+        }
+        if (null != statsExporter) {
+            try {
+                statsExporter.start();
+            } catch (Exception e) {
+                LOG.error("Fail to start stats exporter : ", e);
+            }
+        }
+    }
+
+    @Override
+    public void stop() {
+        if (null != statsExporter) {
+            try {
+                statsExporter.stop();
+            } catch (Exception e) {
+                LOG.error("Fail to stop stats exporter : ", e);
+            }
+        }
+    }
+
+    @Override
+    public StatsLogger getStatsLogger(String name) {
+        return new TwitterStatsLoggerImpl(name);
+    }
+}



Mime
View raw message