hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r441645 - in /lucene/hadoop/trunk: ./ src/contrib/streaming/src/java/org/apache/hadoop/streaming/
Date Fri, 08 Sep 2006 20:45:04 GMT
Author: cutting
Date: Fri Sep  8 13:45:03 2006
New Revision: 441645

URL: http://svn.apache.org/viewvc?view=rev&rev=441645
Log:
HADOOP-517.  Fix bug in contrib/streaming's end of line detection.  Contributed by Hairong.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
    lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamLineRecordReader.java
    lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/UTF8ByteArrayUtils.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=441645&r1=441644&r2=441645
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Fri Sep  8 13:45:03 2006
@@ -158,6 +158,9 @@
 39. HADOOP-458.  Fix a memory corruption bug in libhdfs.
     (Arun C Murthy via cutting)
 
+40. HADOOP-517.  Fix a contrib/streaming bug in end-of-line detection.
+    (Hairong Kuang via cutting)
+
 
 Release 0.5.0 - 2006-08-04
 

Modified: lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java?view=diff&rev=441645&r1=441644&r2=441645
==============================================================================
--- lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
(original)
+++ lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
Fri Sep  8 13:45:03 2006
@@ -397,7 +397,7 @@
             Text key = new Text();
             Text val = new Text();
             // 3/4 Tool to Hadoop
-            while((answer=UTF8ByteArrayUtils.readLine(clientIn_))!= null) {
+            while((answer=UTF8ByteArrayUtils.readLine((InputStream)clientIn_))!= null) {
                 // 4/4 Hadoop out
                 if(optSideEffect_) {
                     sideEffectOut_.write(answer);
@@ -434,7 +434,7 @@
       byte [] line;
       try {
         long num = 0;
-        while((line=UTF8ByteArrayUtils.readLine(clientErr_)) != null) {
+        while((line=UTF8ByteArrayUtils.readLine((InputStream)clientErr_)) != null) {
           num++;
           String lineStr = new String(line, "UTF-8"); 
           logprintln(lineStr);

Modified: lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamLineRecordReader.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamLineRecordReader.java?view=diff&rev=441645&r1=441644&r2=441645
==============================================================================
--- lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamLineRecordReader.java
(original)
+++ lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamLineRecordReader.java
Fri Sep  8 13:45:03 2006
@@ -99,7 +99,7 @@
                 return false;
         }
         
-        line = UTF8ByteArrayUtils.readLine(in_);
+        line = UTF8ByteArrayUtils.readLine((InputStream)in_);
         if(line==null)
             return false;
         try {

Modified: lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/UTF8ByteArrayUtils.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/UTF8ByteArrayUtils.java?view=diff&rev=441645&r1=441644&r2=441645
==============================================================================
--- lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/UTF8ByteArrayUtils.java
(original)
+++ lucene/hadoop/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/UTF8ByteArrayUtils.java
Fri Sep  8 13:45:03 2006
@@ -16,8 +16,9 @@
 
 package org.apache.hadoop.streaming;
 
-import java.io.DataInputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.PushbackInputStream;
 
 import org.apache.hadoop.io.Text;
 
@@ -70,7 +71,7 @@
      * @return a byte array containing the line 
      * @throws IOException
      */
-    public static byte[] readLine(DataInputStream in) throws IOException {
+    public static byte[] readLine(InputStream in) throws IOException {
       byte [] buf = new byte[128];
       byte [] lineBuffer = buf;
       int room = 128;
@@ -84,9 +85,23 @@
         }
 
         char c = (char)b;
-        if (c == '\r' || c == '\n')
+        if (c == '\n')
           break;
 
+        if (c == '\r') {
+          in.mark(1);
+          int c2 = in.read();
+          if(c2 == -1) {
+              isEOF = true;
+              break;
+          }
+          if (c2 != '\n') {
+            // push it back
+            in.reset();
+          }
+          break;
+        }
+        
         if (--room < 0) {
             buf = new byte[offset + 128];
             room = buf.length - offset - 1;



Mime
View raw message