hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r549933 - in /lucene/hadoop/trunk: ./ src/java/org/apache/hadoop/mapred/
Date Fri, 22 Jun 2007 19:09:13 GMT
Author: cutting
Date: Fri Jun 22 12:09:12 2007
New Revision: 549933

URL: http://svn.apache.org/viewvc?view=rev&rev=549933
Log:
HADOOP-1518.  Add a session id to job metrics, for use by HOD.  Contributed by David Bowen.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=549933&r1=549932&r2=549933
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Fri Jun 22 12:09:12 2007
@@ -239,6 +239,9 @@
  73. HADOOP-1512.  Fix failing TestTextInputFormat on Windows.
      (Senthil Subramanian via nigel)
 
+ 74. HADOOP-1518.  Add a session id to job metrics, for use by HOD.
+     (David Bowen via cutting)
+
 
 Release 0.13.0 - 2007-06-08
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java?view=diff&rev=549933&r1=549932&r2=549933
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobConf.java Fri Jun 22 12:09:12
2007
@@ -597,6 +597,34 @@
   }
   
   /**
+   * Get the user-specified session identifier. The default is the empty string.
+   *
+   * The session identifier is used to tag metric data that is reported to some
+   * performance metrics system via the org.apache.hadoop.metrics API.  The 
+   * session identifier is intended, in particular, for use by Hadoop-On-Demand 
+   * (HOD) which allocates a virtual Hadoop cluster dynamically and transiently. 
+   * HOD will set the session identifier by modifying the hadoop-site.xml file 
+   * before starting the cluster.
+   *
+   * When not running under HOD, this identifer is expected to remain set to 
+   * the empty string.
+   *
+   * @return the session identifier, defaulting to ""
+   */
+  public String getSessionId() {
+      return get("session.id", "");
+  }
+  
+  /**
+   * Set the user-specified session idengifier.  
+   *
+   * @param sessionId the new session id
+   */
+  public void setSessionId(String sessionId) {
+      set("session.id", sessionId);
+  }
+    
+  /**
    * Set the maximum no. of failures of a given job per tasktracker.
    * 
    * @param noFailures maximum no. of failures of a given job per tasktracker.

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?view=diff&rev=549933&r1=549932&r2=549933
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java Fri Jun 22 12:09:12
2007
@@ -152,8 +152,9 @@
                                     System.currentTimeMillis(), jobFile); 
         
     MetricsContext metricsContext = MetricsUtil.getContext("mapred");
-    this.jobMetrics = metricsContext.createRecord("job");
+    this.jobMetrics = MetricsUtil.createRecord(metricsContext, "job");
     this.jobMetrics.setTag("user", conf.getUser());
+    this.jobMetrics.setTag("sessionId", conf.getSessionId());
     this.jobMetrics.setTag("jobName", conf.getJobName());
     this.jobMetrics.setTag("jobId", jobid);
   }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?view=diff&rev=549933&r1=549932&r2=549933
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Fri Jun 22 12:09:12
2007
@@ -424,9 +424,10 @@
     private int numJobsSubmitted = 0;
     private int numJobsCompleted = 0;
       
-    JobTrackerMetrics() {
+    JobTrackerMetrics(JobConf conf) {
       MetricsContext context = MetricsUtil.getContext("mapred");
       metricsRecord = MetricsUtil.createRecord(context, "jobtracker");
+      metricsRecord.setTag("sessionId", conf.getSessionId());
       context.registerUpdater(this);
     }
       
@@ -651,7 +652,7 @@
 
     this.startTime = System.currentTimeMillis();
 
-    myMetrics = new JobTrackerMetrics();
+    myMetrics = new JobTrackerMetrics(jobConf);
     this.expireTrackersThread = new Thread(this.expireTrackers,
                                            "expireTrackers");
     this.expireTrackersThread.start();

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java?view=diff&rev=549933&r1=549932&r2=549933
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java Fri Jun 22 12:09:12
2007
@@ -253,7 +253,7 @@
   public LocalJobRunner(Configuration conf) throws IOException {
     this.fs = FileSystem.get(conf);
     this.conf = conf;
-    myMetrics = new JobTrackerMetrics();
+    myMetrics = new JobTrackerMetrics(new JobConf(conf));
   }
 
   // JobSubmissionProtocol methods

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java?view=diff&rev=549933&r1=549932&r2=549933
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/ReduceTask.java Fri Jun 22 12:09:12
2007
@@ -747,6 +747,7 @@
       this.shuffleMetrics = 
         MetricsUtil.createRecord(metricsContext, "shuffleInput");
       this.shuffleMetrics.setTag("user", conf.getUser());
+      this.shuffleMetrics.setTag("sessionId", conf.getSessionId());
 
       // Seed the random number generator with a reasonably globally unique seed
       long randomSeed = System.nanoTime() + 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?view=diff&rev=549933&r1=549932&r2=549933
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Fri Jun 22 12:09:12
2007
@@ -162,8 +162,10 @@
     private int numCompletedTasks = 0;
       
     TaskTrackerMetrics() {
+      JobConf conf = getJobConf();
       MetricsContext context = MetricsUtil.getContext("mapred");
       metricsRecord = MetricsUtil.createRecord(context, "tasktracker");
+      metricsRecord.setTag("sessionId", conf.getSessionId());
       context.registerUpdater(this);
     }
       
@@ -317,10 +319,10 @@
     int numCopiers = this.fConf.getInt("mapred.reduce.parallel.copies", 5);
     //tweak the probe sample size (make it a function of numCopiers)
     probe_sample_size = Math.max(numCopiers*5, 50);
-        
-        
+    
+    
     this.myMetrics = new TaskTrackerMetrics();
-        
+    
     // port numbers
     this.taskReportPort = this.fConf.getInt("mapred.task.tracker.report.port", 50050);
     // bind address



Mime
View raw message