Return-Path: Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: (qmail 91073 invoked from network); 26 Apr 2010 04:15:52 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 26 Apr 2010 04:15:52 -0000 Received: (qmail 14888 invoked by uid 500); 26 Apr 2010 04:15:51 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 14816 invoked by uid 500); 26 Apr 2010 04:15:51 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 14808 invoked by uid 99); 26 Apr 2010 04:15:50 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 26 Apr 2010 04:15:50 +0000 X-ASF-Spam-Status: No, hits=-1001.3 required=10.0 tests=ALL_TRUSTED,AWL X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 26 Apr 2010 04:15:49 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id EF21A23888DD; Mon, 26 Apr 2010 04:15:07 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r937924 - in /hadoop/mapreduce/trunk: CHANGES.txt src/contrib/mumak/src/java/org/apache/hadoop/mapred/SimulatorJobInProgress.java src/java/org/apache/hadoop/mapred/JobInProgress.java src/java/org/apache/hadoop/mapred/JobTrackerMetricsInst.java Date: Mon, 26 Apr 2010 04:15:07 -0000 To: mapreduce-commits@hadoop.apache.org From: sharad@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20100426041507.EF21A23888DD@eris.apache.org> Author: sharad Date: Mon Apr 26 04:15:07 2010 New Revision: 937924 URL: http://svn.apache.org/viewvc?rev=937924&view=rev Log: MAPREDUCE-1219. Remove job level metrics from jobtracker metrics to ease undue load on jobtracker. Contributed by Sreekanth Ramakrishnan. Modified: hadoop/mapreduce/trunk/CHANGES.txt hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/SimulatorJobInProgress.java hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTrackerMetricsInst.java Modified: hadoop/mapreduce/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=937924&r1=937923&r2=937924&view=diff ============================================================================== --- hadoop/mapreduce/trunk/CHANGES.txt (original) +++ hadoop/mapreduce/trunk/CHANGES.txt Mon Apr 26 04:15:07 2010 @@ -576,6 +576,9 @@ Trunk (unreleased changes) MAPREDUCE-1618. Add missing javadoc to JobStatus::*JobAcls. (Amareshwari Sriramadasu via cdouglas) + MAPREDUCE-1219. Remove job level metrics from jobtracker metrics to ease + undue load on jobtracker. (Sreekanth Ramakrishnan via sharad) + Release 0.21.0 - Unreleased INCOMPATIBLE CHANGES Modified: hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/SimulatorJobInProgress.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/SimulatorJobInProgress.java?rev=937924&r1=937923&r2=937924&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/SimulatorJobInProgress.java (original) +++ hadoop/mapreduce/trunk/src/contrib/mumak/src/java/org/apache/hadoop/mapred/SimulatorJobInProgress.java Mon Apr 26 04:15:07 2010 @@ -80,12 +80,6 @@ public class SimulatorJobInProgress exte this.mapFailuresPercent = conf.getMaxMapTaskFailuresPercent(); this.reduceFailuresPercent = conf.getMaxReduceTaskFailuresPercent(); - MetricsContext metricsContext = MetricsUtil.getContext("mapred"); - this.jobMetrics = MetricsUtil.createRecord(metricsContext, "job"); - this.jobMetrics.setTag("user", conf.getUser()); - this.jobMetrics.setTag("sessionId", conf.getSessionId()); - this.jobMetrics.setTag("jobName", conf.getJobName()); - this.jobMetrics.setTag("jobId", jobid.toString()); this.maxLevel = jobtracker.getNumTaskCacheLevels(); this.anyCacheLevel = this.maxLevel + 1; Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=937924&r1=937923&r2=937924&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java (original) +++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java Mon Apr 26 04:15:07 2010 @@ -239,8 +239,6 @@ public class JobInProgress { Counters jobCounters = new Counters(); - MetricsRecord jobMetrics; - // Maximum no. of fetch-failure notifications after which map task is killed private static final int MAX_FETCH_FAILURES_NOTIFICATIONS = 3; @@ -435,12 +433,6 @@ public class JobInProgress { this.mapFailuresPercent = conf.getMaxMapTaskFailuresPercent(); this.reduceFailuresPercent = conf.getMaxReduceTaskFailuresPercent(); - MetricsContext metricsContext = MetricsUtil.getContext("mapred"); - this.jobMetrics = MetricsUtil.createRecord(metricsContext, "job"); - this.jobMetrics.setTag("user", conf.getUser()); - this.jobMetrics.setTag("sessionId", conf.getSessionId()); - this.jobMetrics.setTag("jobName", conf.getJobName()); - this.jobMetrics.setTag("jobId", this.jobId.toString()); hasSpeculativeMaps = conf.getMapSpeculativeExecution(); hasSpeculativeReduces = conf.getReduceSpeculativeExecution(); this.maxLevel = jobtracker.getNumTaskCacheLevels(); @@ -467,35 +459,6 @@ public class JobInProgress { DelegationTokenRenewal.registerDelegationTokensForRenewal( jobInfo.getJobID(), ts, this.conf); } - - /** - * Called periodically by JobTrackerMetrics to update the metrics for - * this job. - */ - public void updateMetrics() { - Counters counters = getCounters(); - for (Counters.Group group : counters) { - jobMetrics.setTag("group", group.getDisplayName()); - for (Counters.Counter counter : group) { - jobMetrics.setTag("counter", counter.getDisplayName()); - jobMetrics.setMetric("value", (float) counter.getCounter()); - jobMetrics.update(); - } - } - } - - /** - * Called when the job is complete - */ - public void cleanUpMetrics() { - // Deletes all metric data for this job (in internal table in metrics package). - // This frees up RAM and possibly saves network bandwidth, since otherwise - // the metrics package implementation might continue to send these job metrics - // after the job has finished. - jobMetrics.removeTag("group"); - jobMetrics.removeTag("counter"); - jobMetrics.remove(); - } private void printCache (Map> cache) { LOG.info("The taskcache info:"); @@ -3295,7 +3258,6 @@ public class JobInProgress { LOG.warn("Error cleaning up "+profile.getJobID()+": "+e); } - cleanUpMetrics(); // free up the memory used by the data structures this.nonRunningMapCache = null; this.runningMapCache = null; Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTrackerMetricsInst.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTrackerMetricsInst.java?rev=937924&r1=937923&r2=937924&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTrackerMetricsInst.java (original) +++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTrackerMetricsInst.java Mon Apr 26 04:15:07 2010 @@ -154,12 +154,6 @@ class JobTrackerMetricsInst extends JobT numTrackersBlackListed = 0; } metricsRecord.update(); - - if (tracker != null) { - for (JobInProgress jip : tracker.getRunningJobs()) { - jip.updateMetrics(); - } - } } @Override