hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r699115 - in /hadoop/core/branches/branch-0.19: ./ src/contrib/streaming/src/java/org/apache/hadoop/streaming/ src/contrib/streaming/src/test/org/apache/hadoop/streaming/ src/core/org/apache/hadoop/util/ src/mapred/org/apache/hadoop/mapred/...
Date Thu, 25 Sep 2008 22:33:34 GMT
Author: omalley
Date: Thu Sep 25 15:33:33 2008
New Revision: 699115

URL: http://svn.apache.org/viewvc?rev=699115&view=rev
Log:
HADOOP-4269. Fix the deprecation of LineReader by extending the new class
into the old name and deprecating it. Also update the tests to test the 
new class. (cdouglas via omalley)

Modified:
    hadoop/core/branches/branch-0.19/CHANGES.txt
    hadoop/core/branches/branch-0.19/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
    hadoop/core/branches/branch-0.19/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamKeyValUtil.java
    hadoop/core/branches/branch-0.19/src/contrib/streaming/src/java/org/apache/hadoop/streaming/UTF8ByteArrayUtils.java
    hadoop/core/branches/branch-0.19/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamedMerge.java
    hadoop/core/branches/branch-0.19/src/core/org/apache/hadoop/util/LineReader.java
    hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/LineRecordReader.java
    hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/lib/NLineInputFormat.java
    hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
    hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestTextInputFormat.java

Modified: hadoop/core/branches/branch-0.19/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/CHANGES.txt?rev=699115&r1=699114&r2=699115&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.19/CHANGES.txt Thu Sep 25 15:33:33 2008
@@ -727,6 +727,10 @@
     HADOOP-4247. Move InputSampler into org.apache.hadoop.mapred.lib, so that
     examples.jar doesn't depend on tools.jar. (omalley)
 
+    HADOOP-4269. Fix the deprecation of LineReader by extending the new class
+    into the old name and deprecating it. Also update the tests to test the 
+    new class. (cdouglas via omalley)
+
 Release 0.18.2 - Unreleased
 
   BUG FIXES

Modified: hadoop/core/branches/branch-0.19/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java?rev=699115&r1=699114&r2=699115&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
(original)
+++ hadoop/core/branches/branch-0.19/src/contrib/streaming/src/java/org/apache/hadoop/streaming/PipeMapRed.java
Thu Sep 25 15:33:33 2008
@@ -34,7 +34,7 @@
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.LineRecordReader.LineReader;
+import org.apache.hadoop.util.LineReader;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.UTF8ByteArrayUtils;
 

Modified: hadoop/core/branches/branch-0.19/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamKeyValUtil.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamKeyValUtil.java?rev=699115&r1=699114&r2=699115&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamKeyValUtil.java
(original)
+++ hadoop/core/branches/branch-0.19/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamKeyValUtil.java
Thu Sep 25 15:33:33 2008
@@ -21,7 +21,7 @@
 import java.io.IOException;
 
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.LineRecordReader.LineReader;
+import org.apache.hadoop.util.LineReader;
 
 public class StreamKeyValUtil {
 

Modified: hadoop/core/branches/branch-0.19/src/contrib/streaming/src/java/org/apache/hadoop/streaming/UTF8ByteArrayUtils.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/streaming/src/java/org/apache/hadoop/streaming/UTF8ByteArrayUtils.java?rev=699115&r1=699114&r2=699115&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/contrib/streaming/src/java/org/apache/hadoop/streaming/UTF8ByteArrayUtils.java
(original)
+++ hadoop/core/branches/branch-0.19/src/contrib/streaming/src/java/org/apache/hadoop/streaming/UTF8ByteArrayUtils.java
Thu Sep 25 15:33:33 2008
@@ -21,8 +21,7 @@
 import java.io.IOException;
 
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.LineRecordReader;
-import org.apache.hadoop.mapred.LineRecordReader.LineReader;
+import org.apache.hadoop.util.LineReader;
 
 /**
  * General utils for byte array containing UTF-8 encoded strings
@@ -205,7 +204,7 @@
    * @param out Text to read into
    * @return number of bytes read
    * @deprecated use 
-   * {@link StreamKeyValUtil#readLine(LineRecordReader.LineReader, Text)} 
+   * {@link StreamKeyValUtil#readLine(LineReader, Text)} 
    * @throws IOException
    */
   @Deprecated

Modified: hadoop/core/branches/branch-0.19/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamedMerge.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamedMerge.java?rev=699115&r1=699114&r2=699115&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamedMerge.java
(original)
+++ hadoop/core/branches/branch-0.19/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamedMerge.java
Thu Sep 25 15:33:33 2008
@@ -36,7 +36,7 @@
 import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.LineRecordReader.LineReader;
+import org.apache.hadoop.util.LineReader;
 import org.apache.hadoop.util.ToolRunner;
 
 /**

Modified: hadoop/core/branches/branch-0.19/src/core/org/apache/hadoop/util/LineReader.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/core/org/apache/hadoop/util/LineReader.java?rev=699115&r1=699114&r2=699115&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/core/org/apache/hadoop/util/LineReader.java (original)
+++ hadoop/core/branches/branch-0.19/src/core/org/apache/hadoop/util/LineReader.java Thu Sep
25 15:33:33 2008
@@ -38,12 +38,23 @@
   private int bufferPosn = 0;
 
   /**
+   * Create a line reader that reads from the given stream using the
+   * default buffer-size (64k).
+   * @param in The input stream
+   * @throws IOException
+   */
+  public LineReader(InputStream in) {
+    this(in, DEFAULT_BUFFER_SIZE);
+  }
+
+  /**
    * Create a line reader that reads from the given stream using the 
    * given buffer-size.
-   * @param in
+   * @param in The input stream
+   * @param bufferSize Size of the read buffer
    * @throws IOException
    */
-  LineReader(InputStream in, int bufferSize) {
+  public LineReader(InputStream in, int bufferSize) {
     this.in = in;
     this.bufferSize = bufferSize;
     this.buffer = new byte[this.bufferSize];

Modified: hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/LineRecordReader.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/LineRecordReader.java?rev=699115&r1=699114&r2=699115&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/LineRecordReader.java
(original)
+++ hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/LineRecordReader.java
Thu Sep 25 15:33:33 2008
@@ -52,145 +52,16 @@
    * @deprecated Use {@link org.apache.hadoop.util.LineReader} instead.
    */
   @Deprecated
-  public static class LineReader {
-    private static final int DEFAULT_BUFFER_SIZE = 64 * 1024;
-    private int bufferSize = DEFAULT_BUFFER_SIZE;
-    private InputStream in;
-    private byte[] buffer;
-    // the number of bytes of real data in the buffer
-    private int bufferLength = 0;
-    // the current position in the buffer
-    private int bufferPosn = 0;
-
-    /**
-     * Create a line reader that reads from the given stream using the 
-     * given buffer-size.
-     * @param in
-     * @throws IOException
-     */
+  public static class LineReader extends org.apache.hadoop.util.LineReader {
+    LineReader(InputStream in) {
+      super(in);
+    }
     LineReader(InputStream in, int bufferSize) {
-      this.in = in;
-      this.bufferSize = bufferSize;
-      this.buffer = new byte[this.bufferSize];
+      super(in, bufferSize);
     }
-
-    /**
-     * Create a line reader that reads from the given stream using the
-     * <code>io.file.buffer.size</code> specified in the given
-     * <code>Configuration</code>.
-     * @param in input stream
-     * @param conf configuration
-     * @throws IOException
-     */
     public LineReader(InputStream in, Configuration conf) throws IOException {
-      this(in, conf.getInt("io.file.buffer.size", DEFAULT_BUFFER_SIZE));
+      super(in, conf);
     }
-
-    /**
-     * Fill the buffer with more data.
-     * @return was there more data?
-     * @throws IOException
-     */
-    boolean backfill() throws IOException {
-      bufferPosn = 0;
-      bufferLength = in.read(buffer);
-      return bufferLength > 0;
-    }
-    
-    /**
-     * Close the underlying stream.
-     * @throws IOException
-     */
-    public void close() throws IOException {
-      in.close();
-    }
-    
-    /**
-     * Read from the InputStream into the given Text.
-     * @param str the object to store the given line
-     * @param maxLineLength the maximum number of bytes to store into str.
-     * @param maxBytesToConsume the maximum number of bytes to consume in this call.
-     * @return the number of bytes read including the newline
-     * @throws IOException if the underlying stream throws
-     */
-    public int readLine(Text str, int maxLineLength,
-                        int maxBytesToConsume) throws IOException {
-      str.clear();
-      boolean hadFinalNewline = false;
-      boolean hadFinalReturn = false;
-      boolean hitEndOfFile = false;
-      int startPosn = bufferPosn;
-      long bytesConsumed = 0;
-      outerLoop: while (true) {
-        if (bufferPosn >= bufferLength) {
-          if (!backfill()) {
-            hitEndOfFile = true;
-            break;
-          }
-        }
-        startPosn = bufferPosn;
-        for(; bufferPosn < bufferLength; ++bufferPosn) {
-          switch (buffer[bufferPosn]) {
-          case '\n':
-            hadFinalNewline = true;
-            bufferPosn += 1;
-            break outerLoop;
-          case '\r':
-            if (hadFinalReturn) {
-              // leave this \r in the stream, so we'll get it next time
-              break outerLoop;
-            }
-            hadFinalReturn = true;
-            break;
-          default:
-            if (hadFinalReturn) {
-              break outerLoop;
-            }
-          }        
-        }
-        bytesConsumed += bufferPosn - startPosn;
-        int length = bufferPosn - startPosn - (hadFinalReturn ? 1 : 0);
-        length = (int)Math.min(length, maxLineLength - str.getLength());
-        if (length >= 0) {
-          str.append(buffer, startPosn, length);
-        }
-        if (bytesConsumed >= maxBytesToConsume) {
-          return (int)Math.min(bytesConsumed, (long)Integer.MAX_VALUE);
-        }
-      }
-      int newlineLength = (hadFinalNewline ? 1 : 0) + (hadFinalReturn ? 1 : 0);
-      if (!hitEndOfFile) {
-        bytesConsumed += bufferPosn - startPosn;
-        int length = bufferPosn - startPosn - newlineLength;
-        length = (int)Math.min(length, maxLineLength - str.getLength());
-        if (length > 0) {
-          str.append(buffer, startPosn, length);
-        }
-      }
-      return (int)Math.min(bytesConsumed, (long)Integer.MAX_VALUE);
-    }
-
-    /**
-     * Read from the InputStream into the given Text.
-     * @param str the object to store the given line
-     * @param maxLineLength the maximum number of bytes to store into str.
-     * @return the number of bytes read including the newline
-     * @throws IOException if the underlying stream throws
-     */
-    public int readLine(Text str, int maxLineLength) throws IOException {
-      return readLine(str, maxLineLength, Integer.MAX_VALUE);
-  }
-
-    /**
-     * Read from the InputStream into the given Text.
-     * @param str the object to store the given line
-     * @return the number of bytes read including the newline
-     * @throws IOException if the underlying stream throws
-     */
-    public int readLine(Text str) throws IOException {
-      return readLine(str, Integer.MAX_VALUE, Integer.MAX_VALUE);
-    }
-
   }
 
   public LineRecordReader(Configuration job, 
@@ -228,7 +99,7 @@
   public LineRecordReader(InputStream in, long offset, long endOffset,
                           int maxLineLength) {
     this.maxLineLength = maxLineLength;
-    this.in = new LineReader(in, LineReader.DEFAULT_BUFFER_SIZE);
+    this.in = new LineReader(in);
     this.start = offset;
     this.pos = offset;
     this.end = endOffset;    

Modified: hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/lib/NLineInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/lib/NLineInputFormat.java?rev=699115&r1=699114&r2=699115&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/lib/NLineInputFormat.java
(original)
+++ hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/lib/NLineInputFormat.java
Thu Sep 25 15:33:33 2008
@@ -35,7 +35,7 @@
 import org.apache.hadoop.mapred.LineRecordReader;
 import org.apache.hadoop.mapred.RecordReader;
 import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.LineRecordReader.LineReader;
+import org.apache.hadoop.util.LineReader;
 
 /**
  * NLineInputFormat which splits N lines of input as one split.

Modified: hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java?rev=699115&r1=699114&r2=699115&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
(original)
+++ hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
Thu Sep 25 15:33:33 2008
@@ -26,7 +26,7 @@
 import org.apache.hadoop.fs.*;
 import org.apache.hadoop.io.*;
 import org.apache.hadoop.io.compress.*;
-import org.apache.hadoop.mapred.LineRecordReader.LineReader;
+import org.apache.hadoop.util.LineReader;
 import org.apache.hadoop.util.ReflectionUtils;
 
 public class TestKeyValueTextInputFormat extends TestCase {
@@ -130,7 +130,7 @@
     }
   }
   private LineReader makeStream(String str) throws IOException {
-    return new LineRecordReader.LineReader(new ByteArrayInputStream
+    return new LineReader(new ByteArrayInputStream
                                            (str.getBytes("UTF-8")), 
                                            defaultConf);
   }

Modified: hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestTextInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestTextInputFormat.java?rev=699115&r1=699114&r2=699115&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestTextInputFormat.java
(original)
+++ hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestTextInputFormat.java
Thu Sep 25 15:33:33 2008
@@ -26,7 +26,7 @@
 import org.apache.hadoop.fs.*;
 import org.apache.hadoop.io.*;
 import org.apache.hadoop.io.compress.*;
-import org.apache.hadoop.mapred.LineRecordReader.LineReader;
+import org.apache.hadoop.util.LineReader;
 import org.apache.hadoop.util.ReflectionUtils;
 
 public class TestTextInputFormat extends TestCase {
@@ -128,7 +128,7 @@
   }
 
   private static LineReader makeStream(String str) throws IOException {
-    return new LineRecordReader.LineReader(new ByteArrayInputStream
+    return new LineReader(new ByteArrayInputStream
                                              (str.getBytes("UTF-8")), 
                                            defaultConf);
   }



Mime
View raw message