hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r540720 - in /lucene/hadoop/trunk: ./ src/java/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/metrics/ src/java/org/apache/hadoop/metrics/spi/
Date Tue, 22 May 2007 19:46:17 GMT
Author: cutting
Date: Tue May 22 12:46:16 2007
New Revision: 540720

URL: http://svn.apache.org/viewvc?view=rev&rev=540720
Log:
HADOOP-1406.  Plug a leak in MapReduce's use of metrics.  Contributed by David Bowen.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/MetricsRecord.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/MetricsRecordImpl.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=540720&r1=540719&r2=540720
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Tue May 22 12:46:16 2007
@@ -51,6 +51,9 @@
      number of block locations per request from the namenode.
      (Konstantin Shvachko via cutting)
 
+ 17. HADOOP-1406.  Plug a leak in MapReduce's use of metrics.
+     (David Bowen via cutting)
+
 
 Branch 0.13 (unreleased changes)
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?view=diff&rev=540720&r1=540719&r2=540720
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java Tue May 22 12:46:16
2007
@@ -152,6 +152,7 @@
     this.jobMetrics = metricsContext.createRecord("job");
     this.jobMetrics.setTag("user", conf.getUser());
     this.jobMetrics.setTag("jobName", conf.getJobName());
+    this.jobMetrics.setTag("jobId", jobid);
   }
 
   /**
@@ -172,6 +173,21 @@
       }
     }
   }
+  
+  
+  /**
+   * Called when the job is complete
+   */
+  public void cleanUpMetrics() {
+    // Deletes all metric data for this job (in internal table in metrics package).
+    // This frees up RAM and possibly saves network bandwidth, since otherwise
+    // the metrics package implementation might continue to send these job metrics
+    // after the job has finished.
+    jobMetrics.removeTag("group");
+    jobMetrics.removeTag("counter");
+    jobMetrics.remove();
+  }
+  
     
   /**
    * Construct the splits, etc.  This is invoked from an async
@@ -1035,6 +1051,8 @@
     } catch (IOException e) {
       LOG.warn("Error cleaning up "+profile.getJobId()+": "+e);
     }
+    
+    cleanUpMetrics();
   }
 
   /**

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/MetricsRecord.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/MetricsRecord.java?view=diff&rev=540720&r1=540719&r2=540720
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/MetricsRecord.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/MetricsRecord.java Tue May 22 12:46:16
2007
@@ -111,6 +111,13 @@
   public abstract void setTag(String tagName, byte tagValue);
     
   /**
+   * Removes any tag of the specified name.
+   *
+   * @param tagName name of a tag
+   */
+  public abstract void removeTag(String tagName);
+  
+  /**
    * Sets the named metric to the specified value.
    *
    * @param metricName name of the metric
@@ -198,8 +205,11 @@
   public abstract void update();
     
   /**
-   * Removes, from the buffered data table, the row (if it exists) having tags 
-   * that equal the tags that have been set on this record. 
+   * Removes, from the buffered data table, all rows having tags 
+   * that equal the tags that have been set on this record. For example,
+   * if there are no tags on this record, all rows for this record name
+   * would be removed.  Or, if there is a single tag on this record, then
+   * just rows containing a tag with the same name and value would be removed.
    */
   public abstract void remove();
     

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java?view=diff&rev=540720&r1=540719&r2=540720
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java
Tue May 22 12:46:16 2007
@@ -25,6 +25,7 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
 import java.util.Timer;
@@ -65,7 +66,21 @@
     TagMap(TagMap orig) {
       super(orig);
     }
+    /**
+     * Returns true if this tagmap contains every tag in other.
+     */
+    public boolean containsAll(TagMap other) {
+      for (Map.Entry<String,Object> entry : other.entrySet()) {
+        Object value = get(entry.getKey());
+        if (value == null || !value.equals(entry.getValue())) {
+          // either key does not exist here, or the value is different
+          return false;
+        }
+      }
+      return true;
+    }
   }
+  
   static class MetricMap extends TreeMap<String,Number> {
     private static final long serialVersionUID = -7495051861141631609L;
   }
@@ -367,9 +382,10 @@
   }
     
   /**
-   * Called by MetricsRecordImpl.remove().  Removes any matching row in
+   * Called by MetricsRecordImpl.remove().  Removes all matching rows in
    * the internal table of metric data.  A row matches if it has the same
-   * tag names and tag values.
+   * tag names and values as record, but it may also have additional
+   * tags.
    */    
   protected void remove(MetricsRecordImpl record) {
     String recordName = record.getRecordName();
@@ -377,7 +393,13 @@
         
     RecordMap recordMap = getRecordMap(recordName);
     synchronized (recordMap) {
-      recordMap.remove(tagTable);
+      Iterator<TagMap> it = recordMap.keySet().iterator();
+      while (it.hasNext()) {
+        TagMap rowTags = it.next();
+        if (rowTags.containsAll(tagTable)) {
+          it.remove();
+        }
+      }
     }
   }
     

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/MetricsRecordImpl.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/MetricsRecordImpl.java?view=diff&rev=540720&r1=540719&r2=540720
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/MetricsRecordImpl.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/metrics/spi/MetricsRecordImpl.java Tue
May 22 12:46:16 2007
@@ -103,6 +103,13 @@
   }
     
   /**
+   * Removes any tag of the specified name.
+   */
+  public void removeTag(String tagName) {
+    tagTable.remove(tagName);
+  }
+  
+  /**
    * Sets the named metric to the specified value.
    *
    * @param metricName name of the metric



Mime
View raw message