hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r505507 - in /lucene/hadoop/trunk: CHANGES.txt conf/hadoop-default.xml src/java/org/apache/hadoop/mapred/JobClient.java src/webapps/task/tasklog.jsp
Date Fri, 09 Feb 2007 21:26:18 GMT
Author: cutting
Date: Fri Feb  9 13:26:17 2007
New Revision: 505507

URL: http://svn.apache.org/viewvc?view=rev&rev=505507
Log:
HADOOP-977.  Send task's stdout and stderr to JobClient's stdout and stderr, respectively.
 Contributed by Arun.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/conf/hadoop-default.xml
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java
    lucene/hadoop/trunk/src/webapps/task/tasklog.jsp

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=505507&r1=505506&r2=505507
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Fri Feb  9 13:26:17 2007
@@ -26,6 +26,10 @@
  7. HADOOP-882.  Upgrade to jets3t version 0.5, used by the S3
     FileSystem.  This version supports retries.  (Michael Stack via cutting)
 
+ 8. HADOOP-977.  Send task's stdout and stderr to JobClient's stdout
+    and stderr respectively, with each line tagged by the task's name.
+    (Arun C Murthy via cutting)
+
 
 Branch 0.11 - unreleased
 

Modified: lucene/hadoop/trunk/conf/hadoop-default.xml
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/conf/hadoop-default.xml?view=diff&rev=505507&r1=505506&r2=505507
==============================================================================
--- lucene/hadoop/trunk/conf/hadoop-default.xml (original)
+++ lucene/hadoop/trunk/conf/hadoop-default.xml Fri Feb  9 13:26:17 2007
@@ -699,6 +699,15 @@
   </description>
 </property>
 
+<property>
+  <name>jobclient.output.filter</name>
+  <value>FAILED</value>
+  <description>The filter for controlling the output of the task's userlogs sent
+               to the console of the JobClient. 
+               The permissible options are: NONE, FAILED, SUCCEEDED and ALL.
+  </description>
+</property>
+
 <!-- ipc properties -->
 
 <property>

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java?view=diff&rev=505507&r1=505506&r2=505507
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java Fri Feb  9 13:26:17
2007
@@ -384,7 +384,7 @@
       String lastReport = null;
       final int MAX_RETRIES = 5;
       int retries = MAX_RETRIES;
-      String outputFilterName = job.get("jobclient.output.filter");
+      String outputFilterName = job.get("jobclient.output.filter", "FAILED");
 
       if (null != outputFilterName) {
         try {
@@ -428,19 +428,19 @@
                   if( event.getTaskStatus() == 
                     TaskCompletionEvent.Status.SUCCEEDED){
                     LOG.info(event.toString());
-                    printHttpFile(event.getTaskTrackerHttp());
+                    displayTaskLogs(event.getTaskId(), event.getTaskTrackerHttp());
                   }
                   break; 
                 case FAILED:
                   if( event.getTaskStatus() == 
                     TaskCompletionEvent.Status.FAILED){
                     LOG.info(event.toString());
-                    printHttpFile(event.getTaskTrackerHttp());
+                    displayTaskLogs(event.getTaskId(), event.getTaskTrackerHttp());
                   }
                   break ; 
                 case ALL:
                   LOG.info(event.toString());
-                  printHttpFile(event.getTaskTrackerHttp());
+                  displayTaskLogs(event.getTaskId(), event.getTaskTrackerHttp());
                   break;
                 }
               }
@@ -467,29 +467,33 @@
         jc.close();
       }
     }
+
+    private static void displayTaskLogs(String taskId, String baseUrl)
+    throws IOException {
+      // Copy tasks's stdout of the JobClient
+      getTaskLogs(taskId, new URL(baseUrl+"&filter=stdout"), System.out);
+      
+      // Copy task's stderr to stderr of the JobClient 
+      getTaskLogs(taskId, new URL(baseUrl+"&filter=stderr"), System.err);
+    }
     
-    static void printHttpFile(String httpURL ) throws IOException {
-      boolean good = false;
-      long totalBytes = 0;
-      URL path = new URL(httpURL); 
+    private static void getTaskLogs(String taskId, URL taskLogUrl, 
+            OutputStream out) {
       try {
-        URLConnection connection = path.openConnection();
-        InputStream input = connection.getInputStream();
+        URLConnection connection = taskLogUrl.openConnection();
+        BufferedReader input = 
+          new BufferedReader(new InputStreamReader(connection.getInputStream()));
+        BufferedWriter output = 
+          new BufferedWriter(new OutputStreamWriter(out));
         try {
-          byte[] buffer = new byte[64 * 1024];
-          int len = input.read(buffer);
-          while (len > 0) {
-            totalBytes += len;
-            LOG.info(new String(buffer).trim());
-            len = input.read(buffer);
-          }
-          good = ((int) totalBytes) == connection.getContentLength();
-          if (!good) {
-          LOG.warn("Incomplete task output received for " + path +
-                          " (" + totalBytes + " instead of " + 
-                          connection.getContentLength() + ")");
+          String logData = null;
+          while ((logData = input.readLine()) != null) {
+            if (logData.length() > 0) {
+              output.write(taskId + ": " + logData + "\n");
+              output.flush();
+            }
           }
-        }finally {
+        } finally {
             input.close();
         }
       }catch(IOException ioe){

Modified: lucene/hadoop/trunk/src/webapps/task/tasklog.jsp
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/webapps/task/tasklog.jsp?view=diff&rev=505507&r1=505506&r2=505507
==============================================================================
--- lucene/hadoop/trunk/src/webapps/task/tasklog.jsp (original)
+++ lucene/hadoop/trunk/src/webapps/task/tasklog.jsp Fri Feb  9 13:26:17 2007
@@ -121,9 +121,6 @@
   	  }
   	  gotRequiredData = false;
   	}
-  	if( plainText ) {
-  	  response.setContentLength(bytesRead); 
-  	}
 	String logData = new String(b, 0, bytesRead);
 	out.println(logData);
   } catch (IOException ioe) {



Mime
View raw message