hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r539095 - in /lucene/hadoop/branches/branch-0.13: ./ src/java/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/mapred/lib/aggregate/ src/test/org/apache/hadoop/mapred/
Date Thu, 17 May 2007 19:59:16 GMT
Author: cutting
Date: Thu May 17 12:59:14 2007
New Revision: 539095

URL: http://svn.apache.org/viewvc?view=rev&rev=539095
Log:
Merge -r 539088:539093 from trunk to 0.13 branch.  Fixes: HADOOP-1356, HADOOP-1363, and HADOOP-1368.

Modified:
    lucene/hadoop/branches/branch-0.13/CHANGES.txt
    lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobClient.java
    lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobInProgress.java
    lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobStatus.java
    lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/lib/aggregate/ValueHistogram.java
    lucene/hadoop/branches/branch-0.13/src/test/org/apache/hadoop/mapred/TestAggregates.java

Modified: lucene/hadoop/branches/branch-0.13/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.13/CHANGES.txt?view=diff&rev=539095&r1=539094&r2=539095
==============================================================================
--- lucene/hadoop/branches/branch-0.13/CHANGES.txt (original)
+++ lucene/hadoop/branches/branch-0.13/CHANGES.txt Thu May 17 12:59:14 2007
@@ -390,6 +390,14 @@
 116. HADOOP-1358.  Fix a potential bug when DFSClient calls skipBytes.
      (Hairong Kuang via cutting)
 
+117. HADOOP-1356.  Fix a bug in ValueHistogram.  (Runping Qi via cutting)
+
+118. HADOOP-1363.  Fix locking bug in JobClient#waitForCompletion().
+     (omalley via cutting)
+
+119. HADOOP-1368.  Fix inconsistent synchronization in JobInProgress.
+     (omalley via cutting)
+
 
 Release 0.12.3 - 2007-04-06
 

Modified: lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobClient.java?view=diff&rev=539095&r1=539094&r2=539095
==============================================================================
--- lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobClient.java (original)
+++ lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobClient.java Thu
May 17 12:59:14 2007
@@ -137,7 +137,7 @@
     /**
      * Blocks until the job is finished
      */
-    public synchronized void waitForCompletion() throws IOException {
+    public void waitForCompletion() throws IOException {
       while (!isComplete()) {
         try {
           Thread.sleep(5000);

Modified: lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobInProgress.java?view=diff&rev=539095&r1=539094&r2=539095
==============================================================================
--- lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobInProgress.java
Thu May 17 12:59:14 2007
@@ -220,7 +220,9 @@
       // Finished time need to be setted here to prevent this job to be retired
       // from the job tracker jobs at the next retire iteration.
       this.finishTime = System.currentTimeMillis();
-      this.status = new JobStatus(status.getJobId(), 1.0f, 1.0f, JobStatus.SUCCEEDED);
+      status.setMapProgress(1.0f);
+      status.setReduceProgress(1.0f);
+      status.setRunState(JobStatus.SUCCEEDED);
       tasksInited = true;
 
       // Special case because the Job is not queued
@@ -263,7 +265,7 @@
   public int desiredMaps() {
     return numMapTasks;
   }
-  public int finishedMaps() {
+  public synchronized int finishedMaps() {
     return finishedMapTasks;
   }
   public int desiredReduces() {
@@ -275,7 +277,7 @@
   public synchronized int runningReduces() {
     return runningReduceTasks;
   }
-  public int finishedReduces() {
+  public synchronized int finishedReduces() {
     return finishedReduceTasks;
   }
  
@@ -485,8 +487,9 @@
   /**
    * Return a MapTask, if appropriate, to run on the given tasktracker
    */
-  public Task obtainNewMapTask(TaskTrackerStatus tts, int clusterSize
-                               ) throws IOException {
+  public synchronized Task obtainNewMapTask(TaskTrackerStatus tts, 
+                                            int clusterSize
+                                           ) throws IOException {
     if (!tasksInited) {
       LOG.info("Cannot create task split for " + profile.getJobId());
       return null;
@@ -513,8 +516,9 @@
    * We don't have cache-sensitivity for reduce tasks, as they
    *  work on temporary MapRed files.  
    */
-  public Task obtainNewReduceTask(TaskTrackerStatus tts,
-                                  int clusterSize) throws IOException {
+  public synchronized Task obtainNewReduceTask(TaskTrackerStatus tts,
+                                               int clusterSize
+                                              ) throws IOException {
     if (!tasksInited) {
       LOG.info("Cannot create task split for " + profile.getJobId());
       return null;

Modified: lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobStatus.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobStatus.java?view=diff&rev=539095&r1=539094&r2=539095
==============================================================================
--- lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobStatus.java (original)
+++ lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/JobStatus.java Thu
May 17 12:59:14 2007
@@ -30,8 +30,6 @@
  * Describes the current status of a job.  This is
  * not intended to be a comprehensive piece of data.
  * For that, look at JobProfile.
- *
- * @author Mike Cafarella
  **************************************************/
 public class JobStatus implements Writable {
 
@@ -83,39 +81,38 @@
   /**
    * @return Percentage of progress in maps 
    */
-  public float mapProgress() { return mapProgress; }
+  public synchronized float mapProgress() { return mapProgress; }
     
   /**
    * Sets the map progress of this job
    * @param p The value of map progress to set to
    */
-  void setMapProgress(float p) { 
+  synchronized void setMapProgress(float p) { 
     this.mapProgress = (float) Math.min(1.0, Math.max(0.0, p)); 
-    
   }
     
   /**
    * @return Percentage of progress in reduce 
    */
-  public float reduceProgress() { return reduceProgress; }
+  public synchronized float reduceProgress() { return reduceProgress; }
     
   /**
    * Sets the reduce progress of this Job
    * @param p The value of reduce progress to set to
    */
-  void setReduceProgress(float p) { 
+  synchronized void setReduceProgress(float p) { 
     this.reduceProgress = (float) Math.min(1.0, Math.max(0.0, p)); 
   }
     
   /**
    * @return running state of the job
    */
-  public int getRunState() { return runState; }
+  public synchronized int getRunState() { return runState; }
     
   /**
    * Change the current run state of the job.
    */
-  public void setRunState(int state) {
+  public synchronized void setRunState(int state) {
     this.runState = state;
   }
 
@@ -123,22 +120,22 @@
    * Set the start time of the job
    * @param startTime The startTime of the job
    */
-  void setStartTime(long startTime) { this.startTime = startTime;};
+  synchronized void setStartTime(long startTime) { this.startTime = startTime;}
     
   /**
    * @return start time of the job
    */
-  public long getStartTime() { return startTime;};
+  synchronized public long getStartTime() { return startTime;}
 
   /**
    * @param user The username of the job
    */
-  void setUsername(String userName) { this.user = userName;};
+  synchronized void setUsername(String userName) { this.user = userName;}
 
   /**
    * @return the username of the job
    */
-  public String getUsername() { return this.user;};
+  public synchronized String getUsername() { return this.user;}
     
   ///////////////////////////////////////
   // Writable

Modified: lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/lib/aggregate/ValueHistogram.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/lib/aggregate/ValueHistogram.java?view=diff&rev=539095&r1=539094&r2=539095
==============================================================================
--- lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/lib/aggregate/ValueHistogram.java
(original)
+++ lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/mapred/lib/aggregate/ValueHistogram.java
Thu May 17 12:59:14 2007
@@ -51,7 +51,7 @@
     String valStr = valCountStr;
     String countStr = "1";
     if (pos >= 0) {
-      valCountStr.substring(0, pos);
+      valStr = valCountStr.substring(0, pos);
       countStr = valCountStr.substring(pos + 1);
     }
     

Modified: lucene/hadoop/branches/branch-0.13/src/test/org/apache/hadoop/mapred/TestAggregates.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.13/src/test/org/apache/hadoop/mapred/TestAggregates.java?view=diff&rev=539095&r1=539094&r2=539095
==============================================================================
--- lucene/hadoop/branches/branch-0.13/src/test/org/apache/hadoop/mapred/TestAggregates.java
(original)
+++ lucene/hadoop/branches/branch-0.13/src/test/org/apache/hadoop/mapred/TestAggregates.java
Thu May 17 12:59:14 2007
@@ -106,6 +106,8 @@
     boolean success = true;
     Path outPath = new Path(OUTPUT_DIR, "part-00000");
     String outdata = TestMiniMRWithDFS.readOutput(outPath,job);
+    System.out.println("full out data:");
+    System.out.println(outdata.toString());
     outdata = outdata.substring(0, expectedOutput.toString().length());
 
     assertEquals(expectedOutput.toString(),outdata);



Mime
View raw message