Return-Path: Delivered-To: apmail-hadoop-core-commits-archive@www.apache.org Received: (qmail 73802 invoked from network); 7 Apr 2009 20:40:02 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 7 Apr 2009 20:40:02 -0000 Received: (qmail 37005 invoked by uid 500); 7 Apr 2009 20:40:02 -0000 Delivered-To: apmail-hadoop-core-commits-archive@hadoop.apache.org Received: (qmail 36960 invoked by uid 500); 7 Apr 2009 20:40:02 -0000 Mailing-List: contact core-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: core-dev@hadoop.apache.org Delivered-To: mailing list core-commits@hadoop.apache.org Received: (qmail 36951 invoked by uid 99); 7 Apr 2009 20:40:02 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 07 Apr 2009 20:40:02 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 07 Apr 2009 20:39:52 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 8C15E23888A0; Tue, 7 Apr 2009 20:39:31 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r762947 - in /hadoop/core/trunk: ./ src/ src/core/org/apache/hadoop/ src/core/org/apache/hadoop/http/ src/core/org/apache/hadoop/metrics/ src/core/org/apache/hadoop/metrics/file/ src/core/org/apache/hadoop/metrics/ganglia/ src/core/org/apac... Date: Tue, 07 Apr 2009 20:39:31 -0000 To: core-commits@hadoop.apache.org From: cutting@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20090407203931.8C15E23888A0@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: cutting Date: Tue Apr 7 20:39:30 2009 New Revision: 762947 URL: http://svn.apache.org/viewvc?rev=762947&view=rev Log: HADOOP-5469. Add /metrics servlet to daemons, providing metrics over HTTP as either text or JSON. Contributed by Philip Zeyliger. Modified: hadoop/core/trunk/CHANGES.txt hadoop/core/trunk/src/core/org/apache/hadoop/HadoopVersionAnnotation.java hadoop/core/trunk/src/core/org/apache/hadoop/http/HttpServer.java hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ContextFactory.java hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java hadoop/core/trunk/src/core/org/apache/hadoop/metrics/file/FileContext.java hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ganglia/GangliaContext.java hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/NullContextWithUpdateThread.java hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/OutputRecord.java hadoop/core/trunk/src/core/org/apache/hadoop/util/VersionInfo.java hadoop/core/trunk/src/saveVersion.sh Modified: hadoop/core/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=762947&r1=762946&r2=762947&view=diff ============================================================================== --- hadoop/core/trunk/CHANGES.txt (original) +++ hadoop/core/trunk/CHANGES.txt Tue Apr 7 20:39:30 2009 @@ -80,6 +80,9 @@ HADOOP-5518. Add contrib/mrunit, a MapReduce unit test framework. (Aaron Kimball via cutting) + HADOOP-5469. Add /metrics servlet to daemons, providing metrics + over HTTP as either text or JSON. (Philip Zeyliger via cutting) + IMPROVEMENTS HADOOP-4565. Added CombineFileInputFormat to use data locality information Modified: hadoop/core/trunk/src/core/org/apache/hadoop/HadoopVersionAnnotation.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/HadoopVersionAnnotation.java?rev=762947&r1=762946&r2=762947&view=diff ============================================================================== --- hadoop/core/trunk/src/core/org/apache/hadoop/HadoopVersionAnnotation.java (original) +++ hadoop/core/trunk/src/core/org/apache/hadoop/HadoopVersionAnnotation.java Tue Apr 7 20:39:30 2009 @@ -42,7 +42,7 @@ * @return the date in unix 'date' format */ String date(); - + /** * Get the url for the subversion repository. */ @@ -59,4 +59,11 @@ * @return The branch name, e.g. "trunk" or "branches/branch-0.20" */ String branch(); + + /** + * Get a checksum of the source files from which + * Hadoop was compiled. + * @return a string that uniquely identifies the source + **/ + String srcChecksum(); } Modified: hadoop/core/trunk/src/core/org/apache/hadoop/http/HttpServer.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/http/HttpServer.java?rev=762947&r1=762946&r2=762947&view=diff ============================================================================== --- hadoop/core/trunk/src/core/org/apache/hadoop/http/HttpServer.java (original) +++ hadoop/core/trunk/src/core/org/apache/hadoop/http/HttpServer.java Tue Apr 7 20:39:30 2009 @@ -37,6 +37,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.log.LogLevel; +import org.apache.hadoop.metrics.MetricsServlet; import org.apache.hadoop.util.ReflectionUtils; import org.mortbay.jetty.Connector; @@ -188,6 +189,7 @@ // set up default servlets addServlet("stacks", "/stacks", StackServlet.class); addServlet("logLevel", "/logLevel", LogLevel.Servlet.class); + addServlet("metrics", "/metrics", MetricsServlet.class); } public void addContext(Context ctxt, boolean isFiltered) Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ContextFactory.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ContextFactory.java?rev=762947&r1=762946&r2=762947&view=diff ============================================================================== --- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ContextFactory.java (original) +++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ContextFactory.java Tue Apr 7 20:39:30 2009 @@ -22,6 +22,8 @@ import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -142,6 +144,14 @@ IllegalAccessException { return getContext(contextName, contextName); } + + /** + * Returns all MetricsContexts built by this factory. + */ + public synchronized Collection getAllContexts() { + // Make a copy to avoid race conditions with creating new contexts. + return new ArrayList(contextMap.values()); + } /** * Returns a "null" context - one which does nothing. Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java?rev=762947&r1=762946&r2=762947&view=diff ============================================================================== --- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java (original) +++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java Tue Apr 7 20:39:30 2009 @@ -21,6 +21,10 @@ package org.apache.hadoop.metrics; import java.io.IOException; +import java.util.Collection; +import java.util.Map; + +import org.apache.hadoop.metrics.spi.OutputRecord; /** * The main interface to the metrics package. @@ -103,5 +107,12 @@ * Returns the timer period. */ public abstract int getPeriod(); - + + /** + * Retrieves all the records managed by this MetricsContext. + * Useful for monitoring systems that are polling-based. + * + * @return A non-null map from all record names to the records managed. + */ + Map> getAllRecords(); } Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/file/FileContext.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/file/FileContext.java?rev=762947&r1=762946&r2=762947&view=diff ============================================================================== --- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/file/FileContext.java (original) +++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/file/FileContext.java Tue Apr 7 20:39:30 2009 @@ -27,7 +27,6 @@ import java.io.PrintWriter; import org.apache.hadoop.metrics.ContextFactory; -import org.apache.hadoop.metrics.MetricsException; import org.apache.hadoop.metrics.spi.AbstractMetricsContext; import org.apache.hadoop.metrics.spi.OutputRecord; @@ -62,18 +61,7 @@ file = new File(fileName); } - String periodStr = getAttribute(PERIOD_PROPERTY); - if (periodStr != null) { - int period = 0; - try { - period = Integer.parseInt(periodStr); - } catch (NumberFormatException nfe) { - } - if (period <= 0) { - throw new MetricsException("Invalid period: " + periodStr); - } - setPeriod(period); - } + parseAndSetPeriod(PERIOD_PROPERTY); } /** Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ganglia/GangliaContext.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ganglia/GangliaContext.java?rev=762947&r1=762946&r2=762947&view=diff ============================================================================== --- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ganglia/GangliaContext.java (original) +++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ganglia/GangliaContext.java Tue Apr 7 20:39:30 2009 @@ -88,19 +88,7 @@ public void init(String contextName, ContextFactory factory) { super.init(contextName, factory); - - String periodStr = getAttribute(PERIOD_PROPERTY); - if (periodStr != null) { - int period = 0; - try { - period = Integer.parseInt(periodStr); - } catch (NumberFormatException nfe) { - } - if (period <= 0) { - throw new MetricsException("Invalid period: " + periodStr); - } - setPeriod(period); - } + parseAndSetPeriod(PERIOD_PROPERTY); metricsServers = Util.parse(getAttribute(SERVERS_PROPERTY), DEFAULT_PORT); Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java?rev=762947&r1=762946&r2=762947&view=diff ============================================================================== --- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java (original) +++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java Tue Apr 7 20:39:30 2009 @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.Timer; @@ -60,7 +61,7 @@ private ContextFactory factory = null; private String contextName = null; - static class TagMap extends TreeMap { + public static class TagMap extends TreeMap { private static final long serialVersionUID = 3546309335061952993L; TagMap() { super(); @@ -83,8 +84,14 @@ } } - static class MetricMap extends TreeMap { + public static class MetricMap extends TreeMap { private static final long serialVersionUID = -7495051861141631609L; + MetricMap() { + super(); + } + MetricMap(MetricMap orig) { + super(orig); + } } static class RecordMap extends HashMap { @@ -309,6 +316,28 @@ } flush(); } + + /** + * Retrieves all the records managed by this MetricsContext. + * Useful for monitoring systems that are polling-based. + * @return A non-null collection of all monitoring records. + */ + public synchronized Map> getAllRecords() { + Map> out = new TreeMap>(); + for (String recordName : bufferedData.keySet()) { + RecordMap recordMap = bufferedData.get(recordName); + synchronized (recordMap) { + List records = new ArrayList(); + Set> entrySet = recordMap.entrySet(); + for (Entry entry : entrySet) { + OutputRecord outRec = new OutputRecord(entry.getKey(), entry.getValue()); + records.add(outRec); + } + out.put(recordName, records); + } + } + return out; + } /** * Sends a record to the metrics system. @@ -424,4 +453,23 @@ protected void setPeriod(int period) { this.period = period; } + + /** + * If a period is set in the attribute passed in, override + * the default with it. + */ + protected void parseAndSetPeriod(String attributeName) { + String periodStr = getAttribute(attributeName); + if (periodStr != null) { + int period = 0; + try { + period = Integer.parseInt(periodStr); + } catch (NumberFormatException nfe) { + } + if (period <= 0) { + throw new MetricsException("Invalid period: " + periodStr); + } + setPeriod(period); + } + } } Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/NullContextWithUpdateThread.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/NullContextWithUpdateThread.java?rev=762947&r1=762946&r2=762947&view=diff ============================================================================== --- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/NullContextWithUpdateThread.java (original) +++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/NullContextWithUpdateThread.java Tue Apr 7 20:39:30 2009 @@ -44,21 +44,7 @@ public void init(String contextName, ContextFactory factory) { super.init(contextName, factory); - - // If period is specified, use it, otherwise the default is good enough - - String periodStr = getAttribute(PERIOD_PROPERTY); - if (periodStr != null) { - int period = 0; - try { - period = Integer.parseInt(periodStr); - } catch (NumberFormatException nfe) { - } - if (period <= 0) { - throw new MetricsException("Invalid period: " + periodStr); - } - setPeriod(period); - } + parseAndSetPeriod(PERIOD_PROPERTY); } Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/OutputRecord.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/OutputRecord.java?rev=762947&r1=762946&r2=762947&view=diff ============================================================================== --- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/OutputRecord.java (original) +++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/OutputRecord.java Tue Apr 7 20:39:30 2009 @@ -21,7 +21,11 @@ package org.apache.hadoop.metrics.spi; import java.util.Collections; +import java.util.Map; import java.util.Set; +import java.util.TreeMap; +import java.util.Map.Entry; + import org.apache.hadoop.metrics.spi.AbstractMetricsContext.MetricMap; import org.apache.hadoop.metrics.spi.AbstractMetricsContext.TagMap; @@ -68,5 +72,19 @@ public Number getMetric(String name) { return metricMap.get(name); } - + + + /** + * Returns a copy of this record's tags. + */ + public TagMap getTagsCopy() { + return new TagMap(tagMap); + } + + /** + * Returns a copy of this record's metrics. + */ + public MetricMap getMetricsCopy() { + return new MetricMap(metricMap); + } } Modified: hadoop/core/trunk/src/core/org/apache/hadoop/util/VersionInfo.java URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/util/VersionInfo.java?rev=762947&r1=762946&r2=762947&view=diff ============================================================================== --- hadoop/core/trunk/src/core/org/apache/hadoop/util/VersionInfo.java (original) +++ hadoop/core/trunk/src/core/org/apache/hadoop/util/VersionInfo.java Tue Apr 7 20:39:30 2009 @@ -87,7 +87,15 @@ public static String getUrl() { return version != null ? version.url() : "Unknown"; } - + + /** + * Get the checksum of the source files from which Hadoop was + * built. + **/ + public static String getSrcChecksum() { + return version != null ? version.srcChecksum() : "Unknown"; + } + /** * Returns the buildVersion which includes version, * revision, user and date. @@ -96,12 +104,13 @@ return VersionInfo.getVersion() + " from " + VersionInfo.getRevision() + " by " + VersionInfo.getUser() + - " on " + VersionInfo.getDate(); + " source checksum " + VersionInfo.getSrcChecksum(); } public static void main(String[] args) { System.out.println("Hadoop " + getVersion()); System.out.println("Subversion " + getUrl() + " -r " + getRevision()); System.out.println("Compiled by " + getUser() + " on " + getDate()); + System.out.println("From source with checksum " + getSrcChecksum()); } } Modified: hadoop/core/trunk/src/saveVersion.sh URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/saveVersion.sh?rev=762947&r1=762946&r2=762947&view=diff ============================================================================== --- hadoop/core/trunk/src/saveVersion.sh (original) +++ hadoop/core/trunk/src/saveVersion.sh Tue Apr 7 20:39:30 2009 @@ -41,16 +41,19 @@ branch="Unknown" url="file://$cwd" fi +srcChecksum=`find src -name '*.java' | LC_ALL=C sort | xargs md5sum | md5sum | cut -d ' ' -f 1` + mkdir -p build/src/org/apache/hadoop cat << EOF | \ sed -e "s/VERSION/$version/" -e "s/USER/$user/" -e "s/DATE/$date/" \ -e "s|URL|$url|" -e "s/REV/$revision/" \ - -e "s|BRANCH|$branch|" \ + -e "s|BRANCH|$branch|" -e "s/SRCCHECKSUM/$srcChecksum/" \ > build/src/org/apache/hadoop/package-info.java /* * Generated by src/saveVersion.sh */ @HadoopVersionAnnotation(version="VERSION", revision="REV", branch="BRANCH", - user="USER", date="DATE", url="URL") + user="USER", date="DATE", url="URL", + srcChecksum="SRCCHECKSUM") package org.apache.hadoop; EOF