hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r762947 - in /hadoop/core/trunk: ./ src/ src/core/org/apache/hadoop/ src/core/org/apache/hadoop/http/ src/core/org/apache/hadoop/metrics/ src/core/org/apache/hadoop/metrics/file/ src/core/org/apache/hadoop/metrics/ganglia/ src/core/org/apac...
Date Tue, 07 Apr 2009 20:39:31 GMT
Author: cutting
Date: Tue Apr  7 20:39:30 2009
New Revision: 762947

URL: http://svn.apache.org/viewvc?rev=762947&view=rev
Log:
HADOOP-5469.  Add /metrics servlet to daemons, providing metrics over HTTP as either text
or JSON.  Contributed by Philip Zeyliger.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/core/org/apache/hadoop/HadoopVersionAnnotation.java
    hadoop/core/trunk/src/core/org/apache/hadoop/http/HttpServer.java
    hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ContextFactory.java
    hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java
    hadoop/core/trunk/src/core/org/apache/hadoop/metrics/file/FileContext.java
    hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ganglia/GangliaContext.java
    hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java
    hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/NullContextWithUpdateThread.java
    hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/OutputRecord.java
    hadoop/core/trunk/src/core/org/apache/hadoop/util/VersionInfo.java
    hadoop/core/trunk/src/saveVersion.sh

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Apr  7 20:39:30 2009
@@ -80,6 +80,9 @@
     HADOOP-5518. Add contrib/mrunit, a MapReduce unit test framework.
     (Aaron Kimball via cutting)
 
+    HADOOP-5469.  Add /metrics servlet to daemons, providing metrics
+    over HTTP as either text or JSON.  (Philip Zeyliger via cutting)
+
   IMPROVEMENTS
 
     HADOOP-4565. Added CombineFileInputFormat to use data locality information

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/HadoopVersionAnnotation.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/HadoopVersionAnnotation.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/HadoopVersionAnnotation.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/HadoopVersionAnnotation.java Tue Apr  7 20:39:30
2009
@@ -42,7 +42,7 @@
    * @return the date in unix 'date' format
    */
   String date();
-  
+    
   /**
    * Get the url for the subversion repository.
    */
@@ -59,4 +59,11 @@
    * @return The branch name, e.g. "trunk" or "branches/branch-0.20"
    */
   String branch();
+
+  /**
+   * Get a checksum of the source files from which
+   * Hadoop was compiled.
+   * @return a string that uniquely identifies the source
+   **/
+  String srcChecksum();    
 }

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/http/HttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/http/HttpServer.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/http/HttpServer.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/http/HttpServer.java Tue Apr  7 20:39:30
2009
@@ -37,6 +37,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.log.LogLevel;
+import org.apache.hadoop.metrics.MetricsServlet;
 import org.apache.hadoop.util.ReflectionUtils;
 
 import org.mortbay.jetty.Connector;
@@ -188,6 +189,7 @@
     // set up default servlets
     addServlet("stacks", "/stacks", StackServlet.class);
     addServlet("logLevel", "/logLevel", LogLevel.Servlet.class);
+    addServlet("metrics", "/metrics", MetricsServlet.class);
   }
 
   public void addContext(Context ctxt, boolean isFiltered)

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ContextFactory.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ContextFactory.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ContextFactory.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ContextFactory.java Tue Apr  7 20:39:30
2009
@@ -22,6 +22,8 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
@@ -142,6 +144,14 @@
            IllegalAccessException {
     return getContext(contextName, contextName);
   }
+  
+  /** 
+   * Returns all MetricsContexts built by this factory.
+   */
+  public synchronized Collection<MetricsContext> getAllContexts() {
+    // Make a copy to avoid race conditions with creating new contexts.
+    return new ArrayList<MetricsContext>(contextMap.values());
+  }
     
   /**
    * Returns a "null" context - one which does nothing.

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/MetricsContext.java Tue Apr  7 20:39:30
2009
@@ -21,6 +21,10 @@
 package org.apache.hadoop.metrics;
 
 import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.hadoop.metrics.spi.OutputRecord;
 
 /**
  * The main interface to the metrics package. 
@@ -103,5 +107,12 @@
    * Returns the timer period.
    */
   public abstract int getPeriod();
-    
+  
+  /**
+   * Retrieves all the records managed by this MetricsContext.
+   * Useful for monitoring systems that are polling-based.
+   * 
+   * @return A non-null map from all record names to the records managed.
+   */
+   Map<String, Collection<OutputRecord>> getAllRecords();
 }

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/file/FileContext.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/file/FileContext.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/file/FileContext.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/file/FileContext.java Tue Apr  7
20:39:30 2009
@@ -27,7 +27,6 @@
 import java.io.PrintWriter;
 
 import org.apache.hadoop.metrics.ContextFactory;
-import org.apache.hadoop.metrics.MetricsException;
 import org.apache.hadoop.metrics.spi.AbstractMetricsContext;
 import org.apache.hadoop.metrics.spi.OutputRecord;
 
@@ -62,18 +61,7 @@
       file = new File(fileName);
     }
         
-    String periodStr = getAttribute(PERIOD_PROPERTY);
-    if (periodStr != null) {
-      int period = 0;
-      try {
-        period = Integer.parseInt(periodStr);
-      } catch (NumberFormatException nfe) {
-      }
-      if (period <= 0) {
-        throw new MetricsException("Invalid period: " + periodStr);
-      }
-      setPeriod(period);
-    }
+    parseAndSetPeriod(PERIOD_PROPERTY);
   }
 
   /**

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ganglia/GangliaContext.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ganglia/GangliaContext.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ganglia/GangliaContext.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/ganglia/GangliaContext.java Tue Apr
 7 20:39:30 2009
@@ -88,19 +88,7 @@
     
   public void init(String contextName, ContextFactory factory) {
     super.init(contextName, factory);
-        
-    String periodStr = getAttribute(PERIOD_PROPERTY);
-    if (periodStr != null) {
-      int period = 0;
-      try {
-        period = Integer.parseInt(periodStr);
-      } catch (NumberFormatException nfe) {
-      }
-      if (period <= 0) {
-        throw new MetricsException("Invalid period: " + periodStr);
-      }
-      setPeriod(period);
-    }
+    parseAndSetPeriod(PERIOD_PROPERTY);
         
     metricsServers = 
       Util.parse(getAttribute(SERVERS_PROPERTY), DEFAULT_PORT); 

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java Tue
Apr  7 20:39:30 2009
@@ -26,6 +26,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.Timer;
@@ -60,7 +61,7 @@
   private ContextFactory factory = null;
   private String contextName = null;
     
-  static class TagMap extends TreeMap<String,Object> {
+  public static class TagMap extends TreeMap<String,Object> {
     private static final long serialVersionUID = 3546309335061952993L;
     TagMap() {
       super();
@@ -83,8 +84,14 @@
     }
   }
   
-  static class MetricMap extends TreeMap<String,Number> {
+  public static class MetricMap extends TreeMap<String,Number> {
     private static final long serialVersionUID = -7495051861141631609L;
+    MetricMap() {
+      super();
+    }
+    MetricMap(MetricMap orig) {
+      super(orig);
+    }
   }
             
   static class RecordMap extends HashMap<TagMap,MetricMap> {
@@ -309,6 +316,28 @@
     }
     flush();
   }
+  
+  /**
+   * Retrieves all the records managed by this MetricsContext.
+   * Useful for monitoring systems that are polling-based.
+   * @return A non-null collection of all monitoring records.
+   */
+  public synchronized Map<String, Collection<OutputRecord>> getAllRecords() {
+    Map<String, Collection<OutputRecord>> out = new TreeMap<String, Collection<OutputRecord>>();
+    for (String recordName : bufferedData.keySet()) {
+      RecordMap recordMap = bufferedData.get(recordName);
+      synchronized (recordMap) {
+        List<OutputRecord> records = new ArrayList<OutputRecord>();
+        Set<Entry<TagMap, MetricMap>> entrySet = recordMap.entrySet();
+        for (Entry<TagMap, MetricMap> entry : entrySet) {
+          OutputRecord outRec = new OutputRecord(entry.getKey(), entry.getValue());
+          records.add(outRec);
+        }
+        out.put(recordName, records);
+      }
+    }
+    return out;
+  }
 
   /**
    * Sends a record to the metrics system.
@@ -424,4 +453,23 @@
   protected void setPeriod(int period) {
     this.period = period;
   }
+  
+  /**
+   * If a period is set in the attribute passed in, override
+   * the default with it.
+   */
+  protected void parseAndSetPeriod(String attributeName) {
+    String periodStr = getAttribute(attributeName);
+    if (periodStr != null) {
+      int period = 0;
+      try {
+        period = Integer.parseInt(periodStr);
+      } catch (NumberFormatException nfe) {
+      }
+      if (period <= 0) {
+        throw new MetricsException("Invalid period: " + periodStr);
+      }
+      setPeriod(period);
+    }
+  }
 }

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/NullContextWithUpdateThread.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/NullContextWithUpdateThread.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/NullContextWithUpdateThread.java
(original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/NullContextWithUpdateThread.java
Tue Apr  7 20:39:30 2009
@@ -44,21 +44,7 @@
   
   public void init(String contextName, ContextFactory factory) {
     super.init(contextName, factory);
-    
-    // If period is specified, use it, otherwise the default is good enough
-        
-    String periodStr = getAttribute(PERIOD_PROPERTY);
-    if (periodStr != null) {
-      int period = 0;
-      try {
-        period = Integer.parseInt(periodStr);
-      } catch (NumberFormatException nfe) {
-      }
-      if (period <= 0) {
-        throw new MetricsException("Invalid period: " + periodStr);
-      }
-      setPeriod(period);
-    }
+    parseAndSetPeriod(PERIOD_PROPERTY);
   }
    
     

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/OutputRecord.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/OutputRecord.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/OutputRecord.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/metrics/spi/OutputRecord.java Tue Apr  7
20:39:30 2009
@@ -21,7 +21,11 @@
 package org.apache.hadoop.metrics.spi;
 
 import java.util.Collections;
+import java.util.Map;
 import java.util.Set;
+import java.util.TreeMap;
+import java.util.Map.Entry;
+
 import org.apache.hadoop.metrics.spi.AbstractMetricsContext.MetricMap;
 import org.apache.hadoop.metrics.spi.AbstractMetricsContext.TagMap;
 
@@ -68,5 +72,19 @@
   public Number getMetric(String name) {
     return metricMap.get(name);
   }
-    
+  
+
+  /**
+   * Returns a copy of this record's tags.
+   */
+  public TagMap getTagsCopy() {
+    return new TagMap(tagMap);
+  }
+  
+  /**
+   * Returns a copy of this record's metrics.
+   */
+  public MetricMap getMetricsCopy() {
+    return new MetricMap(metricMap);
+  }
 }

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/util/VersionInfo.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/util/VersionInfo.java?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/util/VersionInfo.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/util/VersionInfo.java Tue Apr  7 20:39:30
2009
@@ -87,7 +87,15 @@
   public static String getUrl() {
     return version != null ? version.url() : "Unknown";
   }
-  
+
+  /**
+   * Get the checksum of the source files from which Hadoop was
+   * built.
+   **/
+  public static String getSrcChecksum() {
+    return version != null ? version.srcChecksum() : "Unknown";
+  }
+
   /**
    * Returns the buildVersion which includes version, 
    * revision, user and date. 
@@ -96,12 +104,13 @@
     return VersionInfo.getVersion() + 
     " from " + VersionInfo.getRevision() +
     " by " + VersionInfo.getUser() + 
-    " on " + VersionInfo.getDate();
+    " source checksum " + VersionInfo.getSrcChecksum();
   }
   
   public static void main(String[] args) {
     System.out.println("Hadoop " + getVersion());
     System.out.println("Subversion " + getUrl() + " -r " + getRevision());
     System.out.println("Compiled by " + getUser() + " on " + getDate());
+    System.out.println("From source with checksum " + getSrcChecksum());
   }
 }

Modified: hadoop/core/trunk/src/saveVersion.sh
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/saveVersion.sh?rev=762947&r1=762946&r2=762947&view=diff
==============================================================================
--- hadoop/core/trunk/src/saveVersion.sh (original)
+++ hadoop/core/trunk/src/saveVersion.sh Tue Apr  7 20:39:30 2009
@@ -41,16 +41,19 @@
   branch="Unknown"
   url="file://$cwd"
 fi
+srcChecksum=`find src -name '*.java' | LC_ALL=C sort | xargs md5sum | md5sum | cut -d ' '
-f 1`
+
 mkdir -p build/src/org/apache/hadoop
 cat << EOF | \
   sed -e "s/VERSION/$version/" -e "s/USER/$user/" -e "s/DATE/$date/" \
       -e "s|URL|$url|" -e "s/REV/$revision/" \
-      -e "s|BRANCH|$branch|" \
+      -e "s|BRANCH|$branch|" -e "s/SRCCHECKSUM/$srcChecksum/" \
       > build/src/org/apache/hadoop/package-info.java
 /*
  * Generated by src/saveVersion.sh
  */
 @HadoopVersionAnnotation(version="VERSION", revision="REV", branch="BRANCH",
-                         user="USER", date="DATE", url="URL")
+                         user="USER", date="DATE", url="URL",
+                         srcChecksum="SRCCHECKSUM")
 package org.apache.hadoop;
 EOF



Mime
View raw message