hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r1098832 - in /hadoop/common/branches/branch-0.20-security-203: CHANGES.txt src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java src/contrib/streaming/src/test/org/apache/hadoop/streaming/StderrApp.java
Date Mon, 02 May 2011 22:52:15 GMT
Author: acmurthy
Date: Mon May  2 22:52:15 2011
New Revision: 1098832

URL: http://svn.apache.org/viewvc?rev=1098832&view=rev
Log:
HADOOP-5623. Ensure streaming status messages aren't overwritten. Contributed by Rick Cox
& Ravi Gummadi.

Modified:
    hadoop/common/branches/branch-0.20-security-203/CHANGES.txt
    hadoop/common/branches/branch-0.20-security-203/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
    hadoop/common/branches/branch-0.20-security-203/src/contrib/streaming/src/test/org/apache/hadoop/streaming/StderrApp.java

Modified: hadoop/common/branches/branch-0.20-security-203/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-203/CHANGES.txt?rev=1098832&r1=1098831&r2=1098832&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-203/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security-203/CHANGES.txt Mon May  2 22:52:15 2011
@@ -1694,6 +1694,9 @@ Release 0.20.2 - Unreleased
     MAPREDUCE-1070. Prevent a deadlock in the fair scheduler servlet.
     (Todd Lipcon via cdouglas)
 
+    HADOOP-5623. Ensure streaming status messages aren't overwritten. (Rick 
+    Cox & Ravi Gummadi via tomwhite)
+    
     MAPREDUCE-1163. Remove unused, hard-coded paths from libhdfs. (Allen
     Wittenauer via cdouglas)
 

Modified: hadoop/common/branches/branch-0.20-security-203/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-203/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java?rev=1098832&r1=1098831&r2=1098832&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-203/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
(original)
+++ hadoop/common/branches/branch-0.20-security-203/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
Mon May  2 22:52:15 2011
@@ -385,7 +385,11 @@ public abstract class PipeMapRed {
           if (now-lastStdoutReport > reporterOutDelay_) {
             lastStdoutReport = now;
             String hline = "Records R/W=" + numRecRead_ + "/" + numRecWritten_;
-            reporter.setStatus(hline);
+            if (!processProvidedStatus_) {
+              reporter.setStatus(hline);
+            } else {
+              reporter.progress();
+            }
             logprintln(hline);
             logflush();
           }
@@ -446,6 +450,7 @@ public abstract class PipeMapRed {
             if (matchesCounter(lineStr)) {
               incrCounter(lineStr);
             } else if (matchesStatus(lineStr)) {
+              processProvidedStatus_ = true;
               setStatus(lineStr);
             } else {
               LOG.warn("Cannot parse reporter line: " + lineStr);
@@ -671,4 +676,5 @@ public abstract class PipeMapRed {
   String LOGNAME;
   PrintStream log_;
 
+  volatile boolean processProvidedStatus_ = false;
 }

Modified: hadoop/common/branches/branch-0.20-security-203/src/contrib/streaming/src/test/org/apache/hadoop/streaming/StderrApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-203/src/contrib/streaming/src/test/org/apache/hadoop/streaming/StderrApp.java?rev=1098832&r1=1098831&r2=1098832&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-203/src/contrib/streaming/src/test/org/apache/hadoop/streaming/StderrApp.java
(original)
+++ hadoop/common/branches/branch-0.20-security-203/src/contrib/streaming/src/test/org/apache/hadoop/streaming/StderrApp.java
Mon May  2 22:52:15 2011
@@ -32,8 +32,16 @@ public class StderrApp
    * postWriteLines to stderr.
    */
   public static void go(int preWriteLines, int sleep, int postWriteLines) throws IOException
{
+    go(preWriteLines, sleep, postWriteLines, false);
+  }
+  
+  public static void go(int preWriteLines, int sleep, int postWriteLines, boolean status)
throws IOException {
     BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
     String line;
+    
+    if (status) {
+      System.err.println("reporter:status:starting echo");
+    }      
        
     while (preWriteLines > 0) {
       --preWriteLines;
@@ -57,13 +65,14 @@ public class StderrApp
 
   public static void main(String[] args) throws IOException {
     if (args.length < 3) {
-      System.err.println("Usage: StderrApp PREWRITE SLEEP POSTWRITE");
+      System.err.println("Usage: StderrApp PREWRITE SLEEP POSTWRITE [STATUS]");
       return;
     }
     int preWriteLines = Integer.parseInt(args[0]);
     int sleep = Integer.parseInt(args[1]);
     int postWriteLines = Integer.parseInt(args[2]);
+    boolean status = args.length > 3 ? Boolean.parseBoolean(args[3]) : false;
     
-    go(preWriteLines, sleep, postWriteLines);
+    go(preWriteLines, sleep, postWriteLines, status);
   }
 }



Mime
View raw message