hadoop-common-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From 坤谷 <kungu....@taobao.com>
Subject IFileOutputStream
Date Mon, 20 Feb 2012 07:53:39 GMT
Hi experts,

I have a question on IFileOutputStream.
I guess when writing key/value length (1 or 2bytes)to IFile, this function is inefficient:
public void write(byte[] b, int off, int len) throws IOException {
  sum.update(b, off,len);
  out.write(b,off,len);
}
I make the following changes, and see some improve under certain circumstance.
Waiting for your professional answer.

Regards,
Kungu



@@ -36,43 +36,128 @@
   private final DataChecksum sum;
   private byte[] barray;
   private boolean closed = false;
+
+  /**
+   * The internal buffer where data is stored.
+   */
+  protected byte buf[];

   /**
-   * Create a checksum output stream that writes
-   * the bytes to the given stream.
-   * @param out
+   * The number of valid bytes in the buffer. This value is always
+   * in the range <tt>0</tt> through <tt>buf.length</tt>; elements
+   * <tt>buf[0]</tt> through <tt>buf[count-1]</tt> contain valid
+   * byte data.
    */
+  protected int count;
+
+  /**
+   * Creates a new checksum buffered output stream to write data to
+   * the specified underlying output stream.
+   *
+   * @param   out   the underlying output stream.
+   */
   public IFileOutputStream(OutputStream out) {
-    super(out);
-    sum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
-        Integer.MAX_VALUE);
-    barray = new byte[sum.getChecksumSize()];
+ this(out, 8192);
   }
-
+
+  /**
+   * Creates a new buffered output stream to write data to the
+   * specified underlying output stream with the specified buffer
+   * size.
+   *
+   * @param   out    the underlying output stream.
+   * @param   size   the buffer size.
+   * @exception IllegalArgumentException if size &lt;= 0.
+   */
+  public IFileOutputStream(OutputStream out, int size) {
+ super(out);
+    if (size <= 0) {
+          throw new IllegalArgumentException("Buffer size <= 0");
+    }
+ buf = new byte[size];
+
+ sum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
+        Integer.MAX_VALUE);
+ barray = new byte[sum.getChecksumSize()];
+  }
+
+  /** Flush the internal buffer */
+  private void flushBuffer() throws IOException {
+      if (count > 0) {
+     sum.update(buf, 0, count);
+    out.write(buf, 0, count);
+    count = 0;
+      }
+  }
+
+  /**
+   * Writes the specified byte to this buffered output stream.
+   *
+   * @param      b   the byte to be written.
+   * @exception  IOException  if an I/O error occurs.
+   */
+  public void write(int b) throws IOException {
+ if (count >= buf.length) {
+    flushBuffer();
+ }
+ buf[count++] = (byte)(b & 0xFF);
+  }
+
+  /**
+   * Writes <code>len</code> bytes from the specified byte array
+   * starting at offset <code>off</code> to this buffered output stream.
+   *
+   * <p> Ordinarily this method stores bytes from the given array into this
+   * stream's buffer, flushing the buffer to the underlying output stream as
+   * needed.  If the requested length is at least as large as this stream's
+   * buffer, however, then this method will flush the buffer and write the
+   * bytes directly to the underlying output stream.  Thus redundant
+   * <code>BufferedOutputStream</code>s will not copy data unnecessarily.
+   *
+   * @param      b     the data.
+   * @param      off   the start offset in the data.
+   * @param      len   the number of bytes to write.
+   * @exception  IOException  if an I/O error occurs.
+   */
+  public void write(byte b[], int off, int len) throws IOException {
+ if (len >= buf.length) {
+    /* If the request length exceeds the size of the output buffer,
+         flush the output buffer and then write the data directly.
+         In this way buffered streams will cascade harmlessly. */
+    flushBuffer();
+    sum.update(b, off, len);
+    out.write(b, off, len);
+    return;
+ }
+ if (len > buf.length - count) {
+    flushBuffer();
+ }
+ System.arraycopy(b, off, buf, count, len);
+ count += len;
+  }
+
+  /**
+   * Flushes this buffered output stream. This forces any buffered
+   * output bytes to be written out to the underlying output stream.
+   *
+   * @exception  IOException  if an I/O error occurs.
+   * @see        java.io.FilterOutputStream#out
+   */
+  public void flush() throws IOException {
+    flushBuffer();
+ out.flush();
+  }
+
   @Override
   public void close() throws IOException {
     if (closed) {
       return;
     }
     closed = true;
+    flushBuffer();
     sum.writeValue(barray, 0, false);
     out.write (barray, 0, sum.getChecksumSize());
     out.flush();
   }
-
-  /**
-   * Write bytes to the stream.
-   */
-  @Override
-  public void write(byte[] b, int off, int len) throws IOException {
-    sum.update(b, off,len);
-    out.write(b,off,len);
-  }
-
-  @Override
-  public void write(int b) throws IOException {
-    barray[0] = (byte) (b & 0xFF);
-    write(barray,0,1);
-  }

 }

________________________________

This email (including any attachments) is confidential and may be legally privileged. If you
received this email in error, please delete it immediately and do not copy it or use it for
any purpose or disclose its contents to any other person. Thank you.

本电邮(包括任何附件)可能含有机密资料并受法律保护。如您不是正确的收件人,请您立即删除本邮件。请不要将本电邮进行复制并用作任何其他用途、或透露本邮件之内容。谢谢。
Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message