Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 2DB6F38B6 for ; Fri, 6 May 2011 07:29:17 +0000 (UTC) Received: (qmail 7206 invoked by uid 500); 6 May 2011 07:29:16 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 6991 invoked by uid 500); 6 May 2011 07:29:16 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 5996 invoked by uid 99); 6 May 2011 07:29:12 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 06 May 2011 07:29:12 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 06 May 2011 07:29:08 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 40A472388AA9; Fri, 6 May 2011 07:28:48 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1100113 [3/5] - in /hadoop/common/trunk: ./ conf/ ivy/ src/java/org/apache/hadoop/metrics/file/ src/java/org/apache/hadoop/metrics2/ src/java/org/apache/hadoop/metrics2/annotation/ src/java/org/apache/hadoop/metrics2/filter/ src/java/org/a... Date: Fri, 06 May 2011 07:28:46 -0000 To: common-commits@hadoop.apache.org From: acmurthy@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110506072848.40A472388AA9@eris.apache.org> Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/Interns.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/Interns.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/Interns.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/Interns.java Fri May 6 07:28:43 2011 @@ -0,0 +1,165 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import java.util.Map; +import java.util.LinkedHashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsTag; + +/** + * Helpers to create interned metrics info + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class Interns { + private static final Log LOG = LogFactory.getLog(Interns.class); + + // A simple intern cache with two keys + // (to avoid creating new (combined) key objects for lookup) + private static abstract class CacheWith2Keys { + private final Map> k1Map = + new LinkedHashMap>() { + private static final long serialVersionUID = 1L; + private boolean gotOverflow = false; + @Override + protected boolean removeEldestEntry(Map.Entry> e) { + boolean overflow = expireKey1At(size()); + if (overflow && !gotOverflow) { + LOG.warn("Metrics intern cache overflow at "+ size() +" for "+ e); + gotOverflow = true; + } + return overflow; + } + }; + + abstract protected boolean expireKey1At(int size); + abstract protected boolean expireKey2At(int size); + abstract protected V newValue(K1 k1, K2 k2); + + synchronized V add(K1 k1, K2 k2) { + Map k2Map = k1Map.get(k1); + if (k2Map == null) { + k2Map = new LinkedHashMap() { + private static final long serialVersionUID = 1L; + private boolean gotOverflow = false; + @Override protected boolean removeEldestEntry(Map.Entry e) { + boolean overflow = expireKey2At(size()); + if (overflow && !gotOverflow) { + LOG.warn("Metrics intern cache overflow at "+ size() +" for "+ e); + gotOverflow = true; + } + return overflow; + } + }; + k1Map.put(k1, k2Map); + } + V v = k2Map.get(k2); + if (v == null) { + v = newValue(k1, k2); + k2Map.put(k2, v); + } + return v; + } + } + + // Sanity limits in case of misuse/abuse. + static final int MAX_INFO_NAMES = 2010; + static final int MAX_INFO_DESCS = 100; // distinct per name + + enum Info { + INSTANCE; + + final CacheWith2Keys cache = + new CacheWith2Keys() { + + @Override protected boolean expireKey1At(int size) { + return size > MAX_INFO_NAMES; + } + + @Override protected boolean expireKey2At(int size) { + return size > MAX_INFO_DESCS; + } + + @Override protected MetricsInfo newValue(String name, String desc) { + return new MetricsInfoImpl(name, desc); + } + }; + } + + /** + * Get a metric info object + * @param name + * @param description + * @return an interned metric info object + */ + public static MetricsInfo info(String name, String description) { + return Info.INSTANCE.cache.add(name, description); + } + + // Sanity limits + static final int MAX_TAG_NAMES = 100; + static final int MAX_TAG_VALUES = 1000; // distinct per name + + enum Tags { + INSTANCE; + + final CacheWith2Keys cache = + new CacheWith2Keys() { + + @Override protected boolean expireKey1At(int size) { + return size > MAX_TAG_NAMES; + } + + @Override protected boolean expireKey2At(int size) { + return size > MAX_TAG_VALUES; + } + + @Override protected MetricsTag newValue(MetricsInfo info, String value) { + return new MetricsTag(info, value); + } + }; + } + + /** + * Get a metrics tag + * @param info of the tag + * @param value of the tag + * @return an interned metrics tag + */ + public static MetricsTag tag(MetricsInfo info, String value) { + return Tags.INSTANCE.cache.add(info, value); + } + + /** + * Get a metrics tag + * @param name of the tag + * @param description of the tag + * @param value of the tag + * @return an interned metrics tag + */ + public static MetricsTag tag(String name, String description, String value) { + return Tags.INSTANCE.cache.add(info(name, description), value); + } +} Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MethodMetric.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MethodMetric.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MethodMetric.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MethodMetric.java Fri May 6 07:28:43 2011 @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import java.lang.reflect.Method; + +import static com.google.common.base.Preconditions.*; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.metrics2.MetricsException; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.annotation.Metric; +import static org.apache.hadoop.metrics2.util.Contracts.*; + +/** + * Metric generated from a method, mostly used by annotation + */ +class MethodMetric extends MutableMetric { + private static final Log LOG = LogFactory.getLog(MethodMetric.class); + + private final Object obj; + private final Method method; + private final MetricsInfo info; + private final MutableMetric impl; + + MethodMetric(Object obj, Method method, MetricsInfo info, Metric.Type type) { + this.obj = checkNotNull(obj, "object"); + this.method = checkArg(method, method.getParameterTypes().length == 0, + "Metric method should have no arguments"); + this.method.setAccessible(true); + this.info = checkNotNull(info, "info"); + impl = newImpl(checkNotNull(type, "metric type")); + } + + private MutableMetric newImpl(Metric.Type metricType) { + Class resType = method.getReturnType(); + switch (metricType) { + case COUNTER: + return newCounter(resType); + case GAUGE: + return newGauge(resType); + case DEFAULT: + return resType == String.class ? newTag(resType) : newGauge(resType); + case TAG: + return newTag(resType); + default: + checkArg(metricType, false, "unsupported metric type"); + return null; + } + } + + MutableMetric newCounter(final Class type) { + if (isInt(type) || isLong(type)) { + return new MutableMetric() { + @Override public void snapshot(MetricsRecordBuilder rb, boolean all) { + try { + Object ret = method.invoke(obj, (Object[])null); + if (isInt(type)) rb.addCounter(info, ((Integer) ret).intValue()); + else rb.addCounter(info, ((Long) ret).longValue()); + } + catch (Exception ex) { + LOG.error("Error invoking method "+ method.getName(), ex); + } + } + }; + } + throw new MetricsException("Unsupported counter type: "+ type.getName()); + } + + static boolean isInt(Class type) { + boolean ret = type == Integer.TYPE || type == Integer.class; + return ret; + } + + static boolean isLong(Class type) { + return type == Long.TYPE || type == Long.class; + } + + static boolean isFloat(Class type) { + return type == Float.TYPE || type == Float.class; + } + + static boolean isDouble(Class type) { + return type == Double.TYPE || type == Double.class; + } + + MutableMetric newGauge(final Class t) { + if (isInt(t) || isLong(t) || isFloat(t) || isDouble(t)) { + return new MutableMetric() { + @Override public void snapshot(MetricsRecordBuilder rb, boolean all) { + try { + Object ret = method.invoke(obj, (Object[]) null); + if (isInt(t)) rb.addGauge(info, ((Integer) ret).intValue()); + else if (isLong(t)) rb.addGauge(info, ((Long) ret).longValue()); + else if (isFloat(t)) rb.addGauge(info, ((Float) ret).floatValue()); + else rb.addGauge(info, ((Double) ret).doubleValue()); + } + catch (Exception ex) { + LOG.error("Error invoking method "+ method.getName(), ex); + } + } + }; + } + throw new MetricsException("Unsupported gauge type: "+ t.getName()); + } + + MutableMetric newTag(Class resType) { + if (resType == String.class) { + return new MutableMetric() { + @Override public void snapshot(MetricsRecordBuilder rb, boolean all) { + try { + Object ret = method.invoke(obj, (Object[]) null); + rb.tag(info, (String) ret); + } + catch (Exception ex) { + LOG.error("Error invoking method "+ method.getName(), ex); + } + } + }; + } + throw new MetricsException("Unsupported tag type: "+ resType.getName()); + } + + @Override public void snapshot(MetricsRecordBuilder builder, boolean all) { + impl.snapshot(builder, all); + } + + static MetricsInfo metricInfo(Method method) { + return Interns.info(nameFrom(method), "Metric for "+ method.getName()); + } + + static String nameFrom(Method method) { + String methodName = method.getName(); + if (methodName.startsWith("get")) { + return StringUtils.capitalize(methodName.substring(3)); + } + return StringUtils.capitalize(methodName); + } +} Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MetricsAnnotations.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MetricsAnnotations.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MetricsAnnotations.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MetricsAnnotations.java Fri May 6 07:28:43 2011 @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.metrics2.MetricsSource; + +/** + * Metrics annotation helpers. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class MetricsAnnotations { + /** + * Make an metrics source from an annotated object. + * @param source the annotated object. + * @return a metrics source + */ + public static MetricsSource makeSource(Object source) { + return new MetricsSourceBuilder(source, + DefaultMetricsFactory.getAnnotatedMetricsFactory()).build(); + } + + public static MetricsSourceBuilder newSourceBuilder(Object source) { + return new MetricsSourceBuilder(source, + DefaultMetricsFactory.getAnnotatedMetricsFactory()); + } +} Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MetricsInfoImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MetricsInfoImpl.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MetricsInfoImpl.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MetricsInfoImpl.java Fri May 6 07:28:43 2011 @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import com.google.common.base.Objects; +import static com.google.common.base.Preconditions.*; +import org.apache.hadoop.metrics2.MetricsInfo; + +/** + * Making implementing metric info a little easier + */ +class MetricsInfoImpl implements MetricsInfo { + private final String name, description; + + MetricsInfoImpl(String name, String description) { + this.name = checkNotNull(name, "name"); + this.description = checkNotNull(description, "description"); + } + + @Override public String name() { + return name; + } + + @Override public String description() { + return description; + } + + @Override public boolean equals(Object obj) { + if (obj instanceof MetricsInfo) { + MetricsInfo other = (MetricsInfo) obj; + return Objects.equal(name, other.name()) && + Objects.equal(description, other.description()); + } + return false; + } + + @Override public int hashCode() { + return Objects.hashCode(name, description); + } + + @Override public String toString() { + return Objects.toStringHelper(this) + .add("name", name).add("description", description) + .toString(); + } +} Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java Fri May 6 07:28:43 2011 @@ -0,0 +1,378 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import java.util.Collection; +import java.util.Map; + +import com.google.common.collect.Maps; +import com.google.common.base.Objects; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsException; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.MetricsTag; +import org.apache.hadoop.metrics2.impl.MsInfo; + +/** + * An optional metrics registry class for creating and maintaining a + * collection of MetricsMutables, making writing metrics source easier. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class MetricsRegistry { + private final Map metricsMap = Maps.newLinkedHashMap(); + private final Map tagsMap = Maps.newLinkedHashMap(); + private final MetricsInfo metricsInfo; + + /** + * Construct the registry with a record name + * @param name of the record of the metrics + */ + public MetricsRegistry(String name) { + metricsInfo = Interns.info(name, name); + } + + /** + * Construct the registry with a metadata object + * @param info the info object for the metrics record/group + */ + public MetricsRegistry(MetricsInfo info) { + metricsInfo = info; + } + + /** + * @return the info object of the metrics registry + */ + public MetricsInfo info() { + return metricsInfo; + } + + /** + * Get a metric by name + * @param name of the metric + * @return the metric object + */ + public synchronized MutableMetric get(String name) { + return metricsMap.get(name); + } + + /** + * Get a tag by name + * @param name of the tag + * @return the tag object + */ + public synchronized MetricsTag getTag(String name) { + return tagsMap.get(name); + } + + /** + * Create a mutable integer counter + * @param name of the metric + * @param desc metric description + * @param iVal initial value + * @return a new counter object + */ + public MutableCounterInt newCounter(String name, String desc, int iVal) { + return newCounter(Interns.info(name, desc), iVal); + } + + /** + * Create a mutable integer counter + * @param info metadata of the metric + * @param iVal initial value + * @return a new counter object + */ + public synchronized MutableCounterInt newCounter(MetricsInfo info, int iVal) { + checkMetricName(info.name()); + MutableCounterInt ret = new MutableCounterInt(info, iVal); + metricsMap.put(info.name(), ret); + return ret; + } + + /** + * Create a mutable long integer counter + * @param name of the metric + * @param desc metric description + * @param iVal initial value + * @return a new counter object + */ + public MutableCounterLong newCounter(String name, String desc, long iVal) { + return newCounter(Interns.info(name, desc), iVal); + } + + /** + * Create a mutable long integer counter + * @param info metadata of the metric + * @param iVal initial value + * @return a new counter object + */ + public synchronized + MutableCounterLong newCounter(MetricsInfo info, long iVal) { + checkMetricName(info.name()); + MutableCounterLong ret = new MutableCounterLong(info, iVal); + metricsMap.put(info.name(), ret); + return ret; + } + + /** + * Create a mutable integer gauge + * @param name of the metric + * @param desc metric description + * @param iVal initial value + * @return a new gauge object + */ + public MutableGaugeInt newGauge(String name, String desc, int iVal) { + return newGauge(Interns.info(name, desc), iVal); + } + /** + * Create a mutable integer gauge + * @param info metadata of the metric + * @param iVal initial value + * @return a new gauge object + */ + public synchronized MutableGaugeInt newGauge(MetricsInfo info, int iVal) { + checkMetricName(info.name()); + MutableGaugeInt ret = new MutableGaugeInt(info, iVal); + metricsMap.put(info.name(), ret); + return ret; + } + + /** + * Create a mutable long integer gauge + * @param name of the metric + * @param desc metric description + * @param iVal initial value + * @return a new gauge object + */ + public MutableGaugeLong newGauge(String name, String desc, long iVal) { + return newGauge(Interns.info(name, desc), iVal); + } + + /** + * Create a mutable long integer gauge + * @param info metadata of the metric + * @param iVal initial value + * @return a new gauge object + */ + public synchronized MutableGaugeLong newGauge(MetricsInfo info, long iVal) { + checkMetricName(info.name()); + MutableGaugeLong ret = new MutableGaugeLong(info, iVal); + metricsMap.put(info.name(), ret); + return ret; + } + + /** + * Create a mutable metric with stats + * @param name of the metric + * @param desc metric description + * @param sampleName of the metric (e.g., "Ops") + * @param valueName of the metric (e.g., "Time" or "Latency") + * @param extended produce extended stat (stdev, min/max etc.) if true. + * @return a new mutable stat metric object + */ + public synchronized MutableStat newStat(String name, String desc, + String sampleName, String valueName, boolean extended) { + checkMetricName(name); + MutableStat ret = + new MutableStat(name, desc, sampleName, valueName, extended); + metricsMap.put(name, ret); + return ret; + } + + /** + * Create a mutable metric with stats + * @param name of the metric + * @param desc metric description + * @param sampleName of the metric (e.g., "Ops") + * @param valueName of the metric (e.g., "Time" or "Latency") + * @return a new mutable metric object + */ + public MutableStat newStat(String name, String desc, + String sampleName, String valueName) { + return newStat(name, desc, sampleName, valueName, false); + } + + /** + * Create a mutable rate metric + * @param name of the metric + * @return a new mutable metric object + */ + public MutableRate newRate(String name) { + return newRate(name, name, false); + } + + /** + * Create a mutable rate metric + * @param name of the metric + * @param description of the metric + * @return a new mutable rate metric object + */ + public MutableRate newRate(String name, String description) { + return newRate(name, description, false); + } + + /** + * Create a mutable rate metric (for throughput measurement) + * @param name of the metric + * @param desc description + * @param extended produce extended stat (stdev/min/max etc.) if true + * @return a new mutable rate metric object + */ + public MutableRate newRate(String name, String desc, boolean extended) { + return newRate(name, desc, extended, true); + } + + @InterfaceAudience.Private + public synchronized MutableRate newRate(String name, String desc, + boolean extended, boolean returnExisting) { + if (returnExisting) { + MutableMetric rate = metricsMap.get(name); + if (rate != null) { + if (rate instanceof MutableRate) return (MutableRate) rate; + throw new MetricsException("Unexpected metrics type "+ rate.getClass() + +" for "+ name); + } + } + checkMetricName(name); + MutableRate ret = new MutableRate(name, desc, extended); + metricsMap.put(name, ret); + return ret; + } + + synchronized void add(String name, MutableMetric metric) { + checkMetricName(name); + metricsMap.put(name, metric); + } + + /** + * Add sample to a stat metric by name. + * @param name of the metric + * @param value of the snapshot to add + */ + public synchronized void add(String name, long value) { + MutableMetric m = metricsMap.get(name); + + if (m != null) { + if (m instanceof MutableStat) { + ((MutableStat) m).add(value); + } + else { + throw new MetricsException("Unsupported add(value) for metric "+ name); + } + } + else { + metricsMap.put(name, newRate(name)); // default is a rate metric + add(name, value); + } + } + + /** + * Set the metrics context tag + * @param name of the context + * @return the registry itself as a convenience + */ + public MetricsRegistry setContext(String name) { + return tag(MsInfo.Context, name, true); + } + + /** + * Add a tag to the metrics + * @param name of the tag + * @param description of the tag + * @param value of the tag + * @return the registry (for keep adding tags) + */ + public MetricsRegistry tag(String name, String description, String value) { + return tag(name, description, value, false); + } + + /** + * Add a tag to the metrics + * @param name of the tag + * @param description of the tag + * @param value of the tag + * @param override existing tag if true + * @return the registry (for keep adding tags) + */ + public MetricsRegistry tag(String name, String description, String value, + boolean override) { + return tag(Interns.info(name, description), value, override); + } + + /** + * Add a tag to the metrics + * @param info metadata of the tag + * @param value of the tag + * @param override existing tag if true + * @return the registry (for keep adding tags etc.) + */ + public synchronized + MetricsRegistry tag(MetricsInfo info, String value, boolean override) { + if (!override) checkTagName(info.name()); + tagsMap.put(info.name(), Interns.tag(info, value)); + return this; + } + + public MetricsRegistry tag(MetricsInfo info, String value) { + return tag(info, value, false); + } + + Collection tags() { + return tagsMap.values(); + } + + Collection metrics() { + return metricsMap.values(); + } + + private void checkMetricName(String name) { + if (metricsMap.containsKey(name)) { + throw new MetricsException("Metric name "+ name +" already exists!"); + } + } + + private void checkTagName(String name) { + if (tagsMap.containsKey(name)) { + throw new MetricsException("Tag "+ name +" already exists!"); + } + } + + /** + * Sample all the mutable metrics and put the snapshot in the builder + * @param builder to contain the metrics snapshot + * @param all get all the metrics even if the values are not changed. + */ + public synchronized void snapshot(MetricsRecordBuilder builder, boolean all) { + for (MetricsTag tag : tags()) { + builder.add(tag); + } + for (MutableMetric metric : metrics()) { + metric.snapshot(builder, all); + } + } + + @Override public String toString() { + return Objects.toStringHelper(this) + .add("info", metricsInfo).add("tags", tags()).add("metrics", metrics()) + .toString(); + } +} Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MetricsSourceBuilder.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MetricsSourceBuilder.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MetricsSourceBuilder.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MetricsSourceBuilder.java Fri May 6 07:28:43 2011 @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Field; +import java.lang.reflect.Method; + +import static com.google.common.base.Preconditions.*; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsException; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.annotation.Metric; +import org.apache.hadoop.metrics2.annotation.Metrics; + +/** + * Helper class to build metrics source object from annotations + */ +@InterfaceAudience.Private +public class MetricsSourceBuilder { + private static final Log LOG = LogFactory.getLog(MetricsSourceBuilder.class); + + private final Object source; + private final MutableMetricsFactory factory; + private final MetricsRegistry registry; + private MetricsInfo info; + private boolean hasAtMetric = false; + private boolean hasRegistry = false; + + MetricsSourceBuilder(Object source, MutableMetricsFactory factory) { + this.source = checkNotNull(source, "source"); + this.factory = checkNotNull(factory, "mutable metrics factory"); + Class cls = source.getClass(); + registry = initRegistry(source); + + for (Field field : cls.getDeclaredFields()) { + add(source, field); + } + for (Method method : cls.getDeclaredMethods()) { + add(source, method); + } + } + + public MetricsSource build() { + if (source instanceof MetricsSource) { + if (hasAtMetric && !hasRegistry) { + throw new MetricsException("Hybrid metrics: registry required."); + } + return (MetricsSource) source; + } + else if (!hasAtMetric) { + throw new MetricsException("No valid @Metric annotation found."); + } + return new MetricsSource() { + @Override + public void getMetrics(MetricsCollector builder, boolean all) { + registry.snapshot(builder.addRecord(registry.info()), all); + } + }; + } + + public MetricsInfo info() { + return info; + } + + private MetricsRegistry initRegistry(Object source) { + Class cls = source.getClass(); + MetricsRegistry r = null; + // Get the registry if it already exists. + for (Field field : cls.getDeclaredFields()) { + if (field.getType() != MetricsRegistry.class) continue; + try { + field.setAccessible(true); + r = (MetricsRegistry) field.get(source); + hasRegistry = r != null; + break; + } + catch (Exception e) { + LOG.warn("Error accessing field "+ field, e); + continue; + } + } + // Create a new registry according to annotation + for (Annotation annotation : cls.getAnnotations()) { + if (annotation instanceof Metrics) { + Metrics ma = (Metrics) annotation; + info = factory.getInfo(cls, ma); + if (r == null) { + r = new MetricsRegistry(info); + } + r.setContext(ma.context()); + } + } + if (r == null) return new MetricsRegistry(cls.getSimpleName()); + return r; + } + + private void add(Object source, Field field) { + for (Annotation annotation : field.getAnnotations()) { + if (!(annotation instanceof Metric)) continue; + try { + // skip fields already set + field.setAccessible(true); + if (field.get(source) != null) continue; + } + catch (Exception e) { + LOG.warn("Error accessing field "+ field +" annotated with"+ + annotation, e); + continue; + } + MutableMetric mutable = factory.newForField(field, (Metric) annotation, + registry); + if (mutable != null) { + try { + field.set(source, mutable); + hasAtMetric = true; + } + catch (Exception e) { + throw new MetricsException("Error setting field "+ field + + " annotated with "+ annotation, e); + } + } + } + } + + private void add(Object source, Method method) { + for (Annotation annotation : method.getAnnotations()) { + if (!(annotation instanceof Metric)) continue; + factory.newForMethod(source, method, (Metric) annotation, registry); + hasAtMetric = true; + } + } +} Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableCounter.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableCounter.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableCounter.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableCounter.java Fri May 6 07:28:43 2011 @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import static com.google.common.base.Preconditions.*; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.metrics2.MetricsInfo; + +/** + * The mutable counter (monotonically increasing) metric interface + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class MutableCounter extends MutableMetric { + private final MetricsInfo info; + + protected MutableCounter(MetricsInfo info) { + this.info = checkNotNull(info, "counter info"); + } + + protected MetricsInfo info() { + return info; + } + + /** + * Increment the metric value by 1. + */ + public abstract void incr(); +} Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableCounterInt.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableCounterInt.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableCounterInt.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableCounterInt.java Fri May 6 07:28:43 2011 @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; + +/** + * A mutable int counter for implementing metrics sources + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class MutableCounterInt extends MutableCounter { + private volatile int value; + + MutableCounterInt(MetricsInfo info, int initValue) { + super(info); + this.value = initValue; + } + + @Override + public synchronized void incr() { + ++value; + setChanged(); + } + + /** + * Increment the value by a delta + * @param delta of the increment + */ + public synchronized void incr(int delta) { + value += delta; + setChanged(); + } + + public int value() { + return value; + } + + @Override + public void snapshot(MetricsRecordBuilder builder, boolean all) { + if (all || changed()) { + builder.addCounter(info(), value); + clearChanged(); + } + } + +} Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableCounterLong.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableCounterLong.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableCounterLong.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableCounterLong.java Fri May 6 07:28:43 2011 @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; + +/** + * A mutable long counter + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class MutableCounterLong extends MutableCounter { + + private volatile long value; + + MutableCounterLong(MetricsInfo info, long initValue) { + super(info); + this.value = initValue; + } + + @Override + public synchronized void incr() { + ++value; + setChanged(); + } + + /** + * Increment the value by a delta + * @param delta of the increment + */ + public synchronized void incr(long delta) { + value += delta; + setChanged(); + } + + public long value() { + return value; + } + + @Override + public void snapshot(MetricsRecordBuilder builder, boolean all) { + if (all || changed()) { + builder.addCounter(info(), value); + clearChanged(); + } + } + +} Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableGauge.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableGauge.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableGauge.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableGauge.java Fri May 6 07:28:43 2011 @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import static com.google.common.base.Preconditions.*; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.metrics2.MetricsInfo; + +/** + * The mutable gauge metric interface + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class MutableGauge extends MutableMetric { + private final MetricsInfo info; + + protected MutableGauge(MetricsInfo info) { + this.info = checkNotNull(info, "metric info"); + } + + protected MetricsInfo info() { + return info; + } + + /** + * Increment the value of the metric by 1 + */ + public abstract void incr(); + + /** + * Decrement the value of the metric by 1 + */ + public abstract void decr(); +} Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableGaugeInt.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableGaugeInt.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableGaugeInt.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableGaugeInt.java Fri May 6 07:28:43 2011 @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; + +/** + * A mutable int gauge + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class MutableGaugeInt extends MutableGauge { + + private volatile int value; + + MutableGaugeInt(MetricsInfo info, int initValue) { + super(info); + this.value = initValue; + } + + public int value() { + return value; + } + + @Override + public synchronized void incr() { + ++value; + setChanged(); + } + + /** + * Increment by delta + * @param delta of the increment + */ + public synchronized void incr(int delta) { + value += delta; + setChanged(); + } + + @Override + public synchronized void decr() { + --value; + setChanged(); + } + + /** + * decrement by delta + * @param delta of the decrement + */ + public synchronized void decr(int delta) { + value -= delta; + setChanged(); + } + + /** + * Set the value of the metric + * @param value to set + */ + public void set(int value) { + this.value = value; + setChanged(); + } + + @Override + public void snapshot(MetricsRecordBuilder builder, boolean all) { + if (all || changed()) { + builder.addGauge(info(), value); + clearChanged(); + } + } +} Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableGaugeLong.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableGaugeLong.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableGaugeLong.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableGaugeLong.java Fri May 6 07:28:43 2011 @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; + +/** + * A mutable long gauge + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class MutableGaugeLong extends MutableGauge { + + private volatile long value; + + MutableGaugeLong(MetricsInfo info, long initValue) { + super(info); + this.value = initValue; + } + + public long value() { + return value; + } + + @Override + public synchronized void incr() { + ++value; + setChanged(); + } + + /** + * Increment by delta + * @param delta of the increment + */ + public synchronized void incr(long delta) { + value += delta; + setChanged(); + } + + @Override + public synchronized void decr() { + --value; + setChanged(); + } + + /** + * decrement by delta + * @param delta of the decrement + */ + public synchronized void decr(long delta) { + value -= delta; + setChanged(); + } + + /** + * Set the value of the metric + * @param value to set + */ + public void set(long value) { + this.value = value; + setChanged(); + } + + public void snapshot(MetricsRecordBuilder builder, boolean all) { + if (all || changed()) { + builder.addGauge(info(), value); + clearChanged(); + } + } + +} Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableMetric.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableMetric.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableMetric.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableMetric.java Fri May 6 07:28:43 2011 @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; + +/** + * The mutable metric interface + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class MutableMetric { + private volatile boolean changed = true; + + /** + * Get a snapshot of the metric + * @param builder the metrics record builder + * @param all if true, snapshot unchanged metrics as well + */ + public abstract void snapshot(MetricsRecordBuilder builder, boolean all); + + /** + * Get a snapshot of metric if changed + * @param builder the metrics record builder + */ + public void snapshot(MetricsRecordBuilder builder) { + snapshot(builder, false); + } + + /** + * Set the changed flag in mutable operations + */ + protected void setChanged() { changed = true; } + + /** + * Clear the changed flag in the snapshot operations + */ + protected void clearChanged() { changed = false; } + + /** + * @return true if metric is changed since last snapshot/snapshot + */ + public boolean changed() { return changed; } +} Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java Fri May 6 07:28:43 2011 @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.metrics2.MetricsException; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.annotation.Metric; +import org.apache.hadoop.metrics2.annotation.Metrics; + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class MutableMetricsFactory { + private static final Log LOG = LogFactory.getLog(MutableMetricsFactory.class); + + MutableMetric newForField(Field field, Metric annotation, + MetricsRegistry registry) { + if (LOG.isDebugEnabled()) { + LOG.debug("field "+ field +" with annotation "+ annotation); + } + MetricsInfo info = getInfo(annotation, field); + MutableMetric metric = newForField(field, annotation); + if (metric != null) { + registry.add(info.name(), metric); + return metric; + } + final Class cls = field.getType(); + if (cls == MutableCounterInt.class) { + return registry.newCounter(info, 0); + } + if (cls == MutableCounterLong.class) { + return registry.newCounter(info, 0L); + } + if (cls == MutableGaugeInt.class) { + return registry.newGauge(info, 0); + } + if (cls == MutableGaugeLong.class) { + return registry.newGauge(info, 0L); + } + if (cls == MutableRate.class) { + return registry.newRate(info.name(), info.description(), + annotation.always()); + } + if (cls == MutableRates.class) { + return new MutableRates(registry); + } + if (cls == MutableStat.class) { + return registry.newStat(info.name(), info.description(), + annotation.sampleName(), annotation.valueName(), + annotation.always()); + } + throw new MetricsException("Unsupported metric field "+ field.getName() + + " of type "+ field.getType().getName()); + } + + MutableMetric newForMethod(Object source, Method method, Metric annotation, + MetricsRegistry registry) { + if (LOG.isDebugEnabled()) { + LOG.debug("method "+ method +" with annotation "+ annotation); + } + MetricsInfo info = getInfo(annotation, method); + MutableMetric metric = newForMethod(source, method, annotation); + metric = metric != null ? metric : + new MethodMetric(source, method, info, annotation.type()); + registry.add(info.name(), metric); + return metric; + } + + /** + * Override to handle custom mutable metrics for fields + * @param field of the metric + * @param annotation of the field + * @return a new metric object or null + */ + protected MutableMetric newForField(Field field, Metric annotation) { + return null; + } + + /** + * Override to handle custom mutable metrics for methods + * @param source the metrics source object + * @param method to return the metric + * @param annotation of the method + * @return a new metric object or null + */ + protected MutableMetric newForMethod(Object source, Method method, + Metric annotation) { + return null; + } + + protected MetricsInfo getInfo(Metric annotation, Field field) { + return getInfo(annotation, getName(field)); + } + + protected String getName(Field field) { + return StringUtils.capitalize(field.getName()); + } + + protected MetricsInfo getInfo(Metric annotation, Method method) { + return getInfo(annotation, getName(method)); + } + + protected MetricsInfo getInfo(Class cls, Metrics annotation) { + String name = annotation.name(); + String about = annotation.about(); + String name2 = name.isEmpty() ? cls.getSimpleName() : name; + return Interns.info(name2, about.isEmpty() ? name2 : about); + } + + protected String getName(Method method) { + String methodName = method.getName(); + if (methodName.startsWith("get")) { + return StringUtils.capitalize(methodName.substring(3)); + } + return StringUtils.capitalize(methodName); + } + + protected MetricsInfo getInfo(Metric annotation, String defaultName) { + String[] value = annotation.value(); + if (value.length == 2) { + return Interns.info(value[0], value[1]); + } + if (value.length == 1) { + return Interns.info(defaultName, value[0]); + } + return Interns.info(defaultName, defaultName); + } +} Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableRate.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableRate.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableRate.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableRate.java Fri May 6 07:28:43 2011 @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * A convenient mutable metric for throughput measurement + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class MutableRate extends MutableStat { + + MutableRate(String name, String description, boolean extended) { + super(name, description, "Ops", "Time", extended); + } +} Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableRates.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableRates.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableRates.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableRates.java Fri May 6 07:28:43 2011 @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import java.lang.reflect.Method; +import java.util.Set; + +import static com.google.common.base.Preconditions.*; +import com.google.common.collect.Sets; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; + +/** + * Helper class to manage a group of mutable rate metrics + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class MutableRates extends MutableMetric { + static final Log LOG = LogFactory.getLog(MutableRates.class); + private final MetricsRegistry registry; + private final Set> protocolCache = Sets.newHashSet(); + + MutableRates(MetricsRegistry registry) { + this.registry = checkNotNull(registry, "metrics registry"); + } + + /** + * Initialize the registry with all the methods in a protocol + * so they all show up in the first snapshot. + * Convenient for JMX implementations. + * @param protocol the protocol class + */ + public void init(Class protocol) { + if (protocolCache.contains(protocol)) return; + protocolCache.add(protocol); + for (Method method : protocol.getDeclaredMethods()) { + String name = method.getName(); + LOG.debug(name); + try { registry.newRate(name, name, false, true); } + catch (Exception e) { + LOG.error("Error creating rate metrics for "+ method.getName(), e); + } + } + } + + /** + * Add a rate sample for a rate metric + * @param name of the rate metric + * @param elapsed time + */ + public void add(String name, long elapsed) { + registry.add(name, elapsed); + } + + @Override + public void snapshot(MetricsRecordBuilder rb, boolean all) { + registry.snapshot(rb, all); + } +} Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableStat.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableStat.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableStat.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/MutableStat.java Fri May 6 07:28:43 2011 @@ -0,0 +1,145 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.util.SampleStat; +import static org.apache.hadoop.metrics2.lib.Interns.*; + +/** + * A mutable metric with stats. + * + * Useful for keeping throughput/latency stats. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class MutableStat extends MutableMetric { + private final MetricsInfo numInfo; + private final MetricsInfo avgInfo; + private final MetricsInfo stdevInfo; + private final MetricsInfo iMinInfo; + private final MetricsInfo iMaxInfo; + private final MetricsInfo minInfo; + private final MetricsInfo maxInfo; + + private final SampleStat intervalStat = new SampleStat(); + private final SampleStat prevStat = new SampleStat(); + private final SampleStat.MinMax minMax = new SampleStat.MinMax(); + private long numSamples = 0; + private boolean extended = false; + + /** + * Construct a sample statistics metric + * @param name of the metric + * @param description of the metric + * @param sampleName of the metric (e.g. "Ops") + * @param valueName of the metric (e.g. "Time", "Latency") + * @param extended create extended stats (stdev, min/max etc.) by default. + */ + public MutableStat(String name, String description, + String sampleName, String valueName, boolean extended) { + String ucName = StringUtils.capitalize(name); + String usName = StringUtils.capitalize(sampleName); + String uvName = StringUtils.capitalize(valueName); + String desc = StringUtils.uncapitalize(description); + String lsName = StringUtils.uncapitalize(sampleName); + String lvName = StringUtils.uncapitalize(valueName); + numInfo = info(ucName +"Num"+ usName, "Number of "+ lsName +" for "+ desc); + avgInfo = info(ucName +"Avg"+ uvName, "Average "+ lvName +" for "+ desc); + stdevInfo = info(ucName +"Stdev"+ uvName, + "Standard deviation of "+ lvName +" for "+ desc); + iMinInfo = info(ucName +"IMin"+ uvName, + "Interval min "+ lvName +" for "+ desc); + iMaxInfo = info(ucName + "IMax"+ uvName, + "Interval max "+ lvName +" for "+ desc); + minInfo = info(ucName +"Min"+ uvName, "Min "+ lvName +" for "+ desc); + maxInfo = info(ucName +"Max"+ uvName, "Max "+ lvName +" for "+ desc); + this.extended = extended; + } + + /** + * Construct a snapshot stat metric with extended stat off by default + * @param name of the metric + * @param description of the metric + * @param sampleName of the metric (e.g. "Ops") + * @param valueName of the metric (e.g. "Time", "Latency") + */ + public MutableStat(String name, String description, + String sampleName, String valueName) { + this(name, description, sampleName, valueName, false); + } + + /** + * Add a number of samples and their sum to the running stat + * @param numSamples number of samples + * @param sum of the samples + */ + public synchronized void add(long numSamples, long sum) { + intervalStat.add(numSamples, sum); + setChanged(); + } + + /** + * Add a snapshot to the metric + * @param value of the metric + */ + public synchronized void add(long value) { + intervalStat.add(value); + minMax.add(value); + setChanged(); + } + + public synchronized void snapshot(MetricsRecordBuilder builder, boolean all) { + if (all || changed()) { + numSamples += intervalStat.numSamples(); + builder.addCounter(numInfo, numSamples) + .addGauge(avgInfo, lastStat().mean()); + if (extended) { + builder.addGauge(stdevInfo, lastStat().stddev()) + .addGauge(iMinInfo, lastStat().min()) + .addGauge(iMaxInfo, lastStat().max()) + .addGauge(minInfo, minMax.min()) + .addGauge(maxInfo, minMax.max()); + } + if (changed()) { + if (numSamples > 0) { + intervalStat.copyTo(prevStat); + intervalStat.reset(); + } + clearChanged(); + } + } + } + + private SampleStat lastStat() { + return changed() ? intervalStat : prevStat; + } + + /** + * Reset the all time min max of the metric + */ + public void resetMinMax() { + minMax.reset(); + } + +} Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/UniqueNames.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/UniqueNames.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/UniqueNames.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/UniqueNames.java Fri May 6 07:28:43 2011 @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import java.util.Map; + +import com.google.common.base.Joiner; +import com.google.common.collect.Maps; + +import org.apache.hadoop.classification.InterfaceAudience; + +/** + * Generates predictable and user-friendly unique names + */ +@InterfaceAudience.Private +public class UniqueNames { + + static class Count { + final String baseName; + int value; + + Count(String name, int value) { + baseName = name; + this.value = value; + } + } + + static final Joiner joiner = Joiner.on('-'); + final Map map = Maps.newHashMap(); + + public synchronized String uniqueName(String name) { + Count c = map.get(name); + if (c == null) { + c = new Count(name, 0); + map.put(name, c); + return name; + } + if (!c.baseName.equals(name)) c = new Count(name, 0); + do { + String newName = joiner.join(name, ++c.value); + Count c2 = map.get(newName); + if (c2 == null) { + map.put(newName, c); + return newName; + } + // handle collisons, assume to be rare cases, + // eg: people explicitly passed in name-\d+ names. + } while (true); + } +} Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/package-info.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/package-info.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/package-info.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/lib/package-info.java Fri May 6 07:28:43 2011 @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A collection of library classes for implementing metrics sources + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +package org.apache.hadoop.metrics2.lib; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file Added: hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/package-info.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/package-info.java?rev=1100113&view=auto ============================================================================== --- hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/package-info.java (added) +++ hadoop/common/trunk/src/java/org/apache/hadoop/metrics2/package-info.java Fri May 6 07:28:43 2011 @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** +

Metrics 2.0

+ +

Overview

+

This package provides a framework for metrics instrumentation + and publication. +

+ +

The framework provides a variety of ways to implement metrics + instrumentation easily via the simple + {@link org.apache.hadoop.metrics2.MetricsSource} interface + or the even simpler and more concise and declarative metrics annotations. + The consumers of metrics just need to implement the simple + {@link org.apache.hadoop.metrics2.MetricsSink} interface. Producers + register the metrics sources with a metrics system, while consumers + register the sinks. A default metrics system is provided to marshal + metrics from sources to sinks based on (per source/sink) configuration + options. All the metrics are also published and queryable via the + standard JMX MBean interface. This document targets the framework users. + Framework developers could also consult the + design + document for architecture and implementation notes. +

+

Sub-packages

+
+
org.apache.hadoop.metrics2.annotation
+
Public annotation interfaces for simpler metrics instrumentation. +
+
org.apache.hadoop.metrics2.impl
+
Implementation classes of the framework for interface and/or + abstract classes defined in the top-level package. Sink plugin code + usually does not need to reference any class here. +
+
org.apache.hadoop.metrics2.lib
+
Convenience classes for implementing metrics sources, including the + Mutable[{@link org.apache.hadoop.metrics2.lib.MutableGauge Gauge}*| + {@link org.apache.hadoop.metrics2.lib.MutableCounter Counter}*| + {@link org.apache.hadoop.metrics2.lib.MutableStat Stat}] and + {@link org.apache.hadoop.metrics2.lib.MetricsRegistry}. +
+
org.apache.hadoop.metrics2.filter
+
Builtin metrics filter implementations include the + {@link org.apache.hadoop.metrics2.filter.GlobFilter} and + {@link org.apache.hadoop.metrics2.filter.RegexFilter}. +
+
org.apache.hadoop.metrics2.source
+
Builtin metrics source implementations including the + {@link org.apache.hadoop.metrics2.source.JvmMetrics}. +
+
org.apache.hadoop.metrics2.sink
+
Builtin metrics sink implementations including the + {@link org.apache.hadoop.metrics2.sink.FileSink}. +
+
org.apache.hadoop.metrics2.util
+
General utilities for implementing metrics sinks etc., including the + {@link org.apache.hadoop.metrics2.util.MetricsCache}. +
+
+ +

Getting started

+

Implementing metrics sources

+ + + + + + + + +
Using annotationsUsing MetricsSource interface
+ 
+  @Metrics(context="MyContext")
+  class MyStat {
+
+    @Metric("My metric description")
+    public int getMyMetric() {
+      return 42;
+    }
+  }
 + 
+  class MyStat implements MetricsSource {
+
+    @Override
+    public void getMetrics(MetricsCollector collector, boolean all) {
+      collector.addRecord("MyStat")
+          .setContext("MyContext")
+          .addGauge(info("MyMetric", "My metric description"), 42);
+    }
+  }
+
+
+

In this example we introduced the following:

+
+
@Metrics
+
The {@link org.apache.hadoop.metrics2.annotation.Metrics} annotation is + used to indicate that the class is a metrics source. +
+ +
MyContext
+
The optional context name typically identifies either the + application, or a group of modules within an application or + library. +
+ +
MyStat
+
The class name is used (by default, or specified by name=value parameter + in the Metrics annotation) as the metrics record name for + which a set of metrics are to be reported. For example, you could have a + record named "CacheStat" for reporting a number of statistics relating to + the usage of some cache in your application.
+ +
@Metric
+
The {@link org.apache.hadoop.metrics2.annotation.Metric} annotation + identifies a particular metric, which in this case, is the + result of the method call getMyMetric of the "gauge" (default) type, + which means it can vary in both directions, compared with a "counter" + type, which can only increase or stay the same. The name of the metric + is "MyMetric" (inferred from getMyMetric method name by default.) The 42 + here is the value of the metric which can be substituted with any valid + java expressions. +
+
+

Note, the {@link org.apache.hadoop.metrics2.MetricsSource} interface is + more verbose but more flexible, + allowing generated metrics names and multiple records. In fact, the + annotation interface is implemented with the MetricsSource interface + internally.

+

Implementing metrics sinks

+ 
+  public class MySink implements MetricsSink {
+    public void putMetrics(MetricsRecord record) {
+      System.out.print(record);
+    }
+    public void init(SubsetConfiguration conf) {}
+    public void flush() {}
+  }
+

In this example there are three additional concepts:

+
+
record
+
This object corresponds to the record created in metrics sources + e.g., the "MyStat" in previous example. +
+
conf
+
The configuration object for the sink instance with prefix removed. + So you can get any sink specific configuration using the usual + get* method. +
+
flush
+
This method is called for each update cycle, which may involve + more than one record. The sink should try to flush any buffered metrics + to its backend upon the call. But it's not required that the + implementation is synchronous. +
+
+

In order to make use our MyMetrics and MySink, + they need to be hooked up to a metrics system. In this case (and most + cases), the DefaultMetricsSystem would suffice. +

+ 
+  DefaultMetricsSystem.initialize("test"); // called once per application
+  DefaultMetricsSystem.register(new MyStat());
+

Metrics system configuration

+

Sinks are usually specified in a configuration file, say, + "hadoop-metrics2-test.properties", as: +

+ 
+  test.sink.mysink0.class=com.example.hadoop.metrics.MySink
+

The configuration syntax is:

+ 
+  [prefix].[source|sink|jmx|].[instance].[option]
+

In the previous example, test is the prefix and + mysink0 is an instance name. + DefaultMetricsSystem would try to load + hadoop-metrics2-[prefix].properties first, and if not found, + try the default hadoop-metrics2.properties in the class path. + Note, the [instance] is an arbitrary name to uniquely + identify a particular sink instance. The asterisk (*) can be + used to specify default options. +

+

Consult the metrics instrumentation in jvm, rpc, hdfs and mapred, etc. + for more examples. +

+ +

Metrics Filtering

+

One of the features of the default metrics system is metrics filtering + configuration by source, context, record/tags and metrics. The least + expensive way to filter out metrics would be at the source level, e.g., + filtering out source named "MyMetrics". The most expensive way would be + per metric filtering. +

+

Here are some examples:

+ 
+  test.sink.file0.class=org.apache.hadoop.metrics2.sink.FileSink
+  test.sink.file0.context=foo
+

In this example, we configured one sink instance that would + accept metrics from context foo only. +

+ 
+  *.source.filter.class=org.apache.hadoop.metrics2.filter.GlobFilter
+  test.*.source.filter.include=foo
+  test.*.source.filter.exclude=bar
+

In this example, we specify a source filter that includes source + foo and excludes bar. When only include + patterns are specified, the filter operates in the white listing mode, + where only matched sources are included. Likewise, when only exclude + patterns are specified, only matched sources are excluded. Sources that + are not matched in either patterns are included as well when both patterns + are present. Note, the include patterns have precedence over the exclude + patterns. +

+

Similarly, you can specify the record.filter and + metrics.filter options, which operate at record and metric + level, respectively. Filters can be combined to optimize + the filtering efficiency.

+ +

Metrics instrumentation strategy

+ + In previous examples, we showed a minimal example to use the + metrics framework. In a larger system (like Hadoop) that allows + custom metrics instrumentation, we recommend the following strategy: + 
+  @Metrics(about="My metrics description", context="MyContext")
+  class MyMetrics extends MyInstrumentation {
+
+    @Metric("My gauge description") MutableGaugeInt gauge0;
+    @Metric("My counter description") MutableCounterLong counter0;
+    @Metric("My rate description") MutableRate rate0;
+
+    @Override public void setGauge0(int value) { gauge0.set(value); }
+    @Override public void incrCounter0() { counter0.incr(); }
+    @Override public void addRate0(long elapsed) { rate0.add(elapsed); }
+  }
+
+ + Note, in this example we introduced the following: +
+
MyInstrumentation
+
This is usually an abstract class (or interface) to define an + instrumentation interface (incrCounter0 etc.) that allows different + implementations. This could be a mechanism to allow different metrics + systems to be used at runtime via configuration. +
+
Mutable[Gauge*|Counter*|Rate]
+
These are library classes to manage mutable metrics for + implementations of metrics sources. They produce immutable gauge and + counters (Metric[Gauge*|Counter*]) for downstream consumption (sinks) + upon snapshot. The MutableRate + in particular, provides a way to measure latency and throughput of an + operation. In this particular case, it produces a long counter + "Rate0NumOps" and double gauge "Rate0AvgTime" when snapshotted. +
+
+ +

Migration from previous system

+

Users of the previous metrics system would notice the lack of + context prefix in the configuration examples. The new + metrics system decouples the concept for context (for grouping) with the + implementation where a particular context object does the updating and + publishing of metrics, which causes problems when you want to have a + single context to be consumed by multiple backends. You would also have to + configure an implementation instance per context, even if you have a + backend that can handle multiple contexts (file, gangalia etc.): +

+ + + + + + + + + + +
BeforeAfter
+  context1.class=org.hadoop.metrics.file.FileContext
+  context2.class=org.hadoop.metrics.file.FileContext
+  ...
+  contextn.class=org.hadoop.metrics.file.FileContext
+ 
+  myprefix.sink.file.class=org.hadoop.metrics2.sink.FileSink
+
+

In the new metrics system, you can simulate the previous behavior by + using the context option in the sink options like the following: +

+ + + + + + + + + + +
BeforeAfter
+  context0.class=org.hadoop.metrics.file.FileContext
+  context0.fileName=context0.out
+  context1.class=org.hadoop.metrics.file.FileContext
+  context1.fileName=context1.out
+  ...
+  contextn.class=org.hadoop.metrics.file.FileContext
+  contextn.fileName=contextn.out
+ 
+  myprefix.sink.*.class=org.apache.hadoop.metrics2.sink.FileSink
+  myprefix.sink.file0.context=context0
+  myprefix.sink.file0.filename=context1.out
+  myprefix.sink.file1.context=context1
+  myprefix.sink.file1.filename=context1.out
+  ...
+  myprefix.sink.filen.context=contextn
+  myprefix.sink.filen.filename=contextn.out
+
+

to send metrics of a particular context to a particular backend. Note, + myprefix is an arbitrary prefix for configuration groupings, + typically they are the name of a particular process + (namenode, jobtracker, etc.) +

+ */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +package org.apache.hadoop.metrics2; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file