hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject svn commit: r1515906 - in /hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs: FSDataInputStream.java FSInputStream.java SupportsZeroCopy.java ZeroCopyCursor.java ZeroCopyUnavailableException.java
Date Tue, 20 Aug 2013 18:07:48 GMT
Author: wang
Date: Tue Aug 20 18:07:47 2013
New Revision: 1515906

URL: http://svn.apache.org/r1515906
Log:
HDFS-4953. Enable HDFS local reads via mmap. Contributed by Colin Patrick McCabe.

Added:
    hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/SupportsZeroCopy.java
  (with props)
    hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyCursor.java
  (with props)
    hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyUnavailableException.java
  (with props)
Modified:
    hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
    hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputStream.java

Modified: hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java?rev=1515906&r1=1515905&r2=1515906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
(original)
+++ hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
Tue Aug 20 18:07:47 2013
@@ -28,9 +28,9 @@ import org.apache.hadoop.classification.
 @InterfaceAudience.Public
 @InterfaceStability.Stable
 public class FSDataInputStream extends DataInputStream
-    implements Seekable, PositionedReadable, Closeable,
-    ByteBufferReadable, HasFileDescriptor, CanSetDropBehind, CanSetReadahead {
-
+    implements Seekable, PositionedReadable, Closeable, 
+      ByteBufferReadable, HasFileDescriptor, CanSetDropBehind, CanSetReadahead,
+      SupportsZeroCopy {
   public FSDataInputStream(InputStream in)
     throws IOException {
     super(in);
@@ -167,4 +167,15 @@ public class FSDataInputStream extends D
           "support setting the drop-behind caching setting.");
     }
   }
+
+  @Override
+  public ZeroCopyCursor createZeroCopyCursor()
+      throws IOException, ZeroCopyUnavailableException {
+    try {
+      return ((SupportsZeroCopy)in).createZeroCopyCursor();
+    }
+    catch (ClassCastException e) {
+      throw new ZeroCopyUnavailableException(e);
+    }
+  }
 }

Modified: hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputStream.java?rev=1515906&r1=1515905&r2=1515906&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputStream.java
(original)
+++ hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputStream.java
Tue Aug 20 18:07:47 2013
@@ -18,9 +18,11 @@
 package org.apache.hadoop.fs;
 
 import java.io.*;
+import java.nio.ByteBuffer;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.ZeroCopyUnavailableException;
 
 /****************************************************************
  * FSInputStream is a generic old InputStream with a little bit
@@ -30,7 +32,7 @@ import org.apache.hadoop.classification.
 @InterfaceAudience.LimitedPrivate({"HDFS"})
 @InterfaceStability.Unstable
 public abstract class FSInputStream extends InputStream
-    implements Seekable, PositionedReadable {
+    implements Seekable, PositionedReadable, SupportsZeroCopy {
   /**
    * Seek to the given offset from the start of the file.
    * The next read() will be from that location.  Can't
@@ -86,4 +88,11 @@ public abstract class FSInputStream exte
     throws IOException {
     readFully(position, buffer, 0, buffer.length);
   }
+
+  @Override
+  public ZeroCopyCursor createZeroCopyCursor() 
+      throws IOException, ZeroCopyUnavailableException {
+    throw new ZeroCopyUnavailableException("zero copy is not implemented " +
+                                           "for this filesystem type.");
+  }
 }

Added: hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/SupportsZeroCopy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/SupportsZeroCopy.java?rev=1515906&view=auto
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/SupportsZeroCopy.java
(added)
+++ hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/SupportsZeroCopy.java
Tue Aug 20 18:07:47 2013
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Supports zero-copy reads.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface SupportsZeroCopy {
+  /**
+   * Get a zero-copy cursor to use for zero-copy reads.
+   *
+   * @throws IOException
+   *     If there was an error creating the ZeroCopyCursor
+   * @throws UnsupportedOperationException
+   *     If this stream does not support zero-copy reads.
+   *     This is used, for example, when one stream wraps another
+   *     which may or may not support ZCR.
+   */
+  public ZeroCopyCursor createZeroCopyCursor()
+      throws IOException, ZeroCopyUnavailableException;
+}
\ No newline at end of file

Propchange: hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/SupportsZeroCopy.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyCursor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyCursor.java?rev=1515906&view=auto
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyCursor.java
(added)
+++ hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyCursor.java
Tue Aug 20 18:07:47 2013
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.Closeable;
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * A ZeroCopyCursor allows you to make zero-copy reads.
+ * 
+ * Cursors should be closed when they are no longer needed.
+ * 
+ * Example:
+ *   FSDataInputStream fis = fs.open("/file");
+ *   ZeroCopyCursor cursor = fis.createZeroCopyCursor();
+ *   try {
+ *     cursor.read(128);
+ *     ByteBuffer data = cursor.getData();
+ *     processData(data);
+ *   } finally {
+ *     cursor.close();
+ *   }
+ */
+public interface ZeroCopyCursor extends Closeable {
+  /**
+   * Set the fallback buffer used for this zero copy cursor.
+   * The fallback buffer is used when a true zero-copy read is impossible.
+   * If there is no fallback buffer, UnsupportedOperationException is thrown
+   * when a true zero-copy read cannot be done.
+   * 
+   * @param fallbackBuffer          The fallback buffer to set, or null for none.
+   */
+  public void setFallbackBuffer(ByteBuffer fallbackBuffer);
+
+  /**
+   * @return the fallback buffer in use, or null if there is none.
+   */
+  public ByteBuffer getFallbackBuffer();
+  
+  /**
+   * @param skipChecksums   Whether we should skip checksumming with this 
+   *                        zero copy cursor.
+   */
+  public void setSkipChecksums(boolean skipChecksums);
+
+  /**
+   * @return                Whether we should skip checksumming with this
+   *                        zero copy cursor.
+   */
+  public boolean getSkipChecksums();
+  
+  /**
+   * @param allowShortReads   Whether we should allow short reads.
+   */
+  public void setAllowShortReads(boolean allowShortReads);
+
+  /**
+   * @return                  Whether we should allow short reads.
+   */
+  public boolean getAllowShortReads();
+
+  /**
+   * Perform a zero-copy read.
+   *
+   * @param toRead          The minimum number of bytes to read.
+   *                        Must not be negative.  If we hit EOF before
+   *                        reading this many bytes, we will either throw
+   *                        EOFException (if allowShortReads = false), or
+   *                        return a short read (if allowShortReads = true).
+   *                        A short read could be as short as 0 bytes.
+   * @throws UnsupportedOperationException
+   *             If a true zero-copy read cannot be done, and no fallback
+   *             buffer was set.
+   * @throws EOFException
+   *             If allowShortReads = false, and we can't read all the bytes
+   *             that were requested.  This will never be thrown if
+   *             allowShortReads = true.
+   * @throws IOException
+   *             If there was an error while reading the data.
+   */
+  public void read(int toRead)
+      throws UnsupportedOperationException, EOFException, IOException;
+
+  /**
+   * Get the current data buffer.
+   *
+   * This buffer will remain valid until either this cursor is closed, or we
+   * call read() again on this same cursor.  You can find the amount of data
+   * that was read previously by calling ByteBuffer#remaining.
+   * 
+   * @return                The current data buffer.
+   */
+  public ByteBuffer getData();
+}

Propchange: hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyCursor.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyUnavailableException.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyUnavailableException.java?rev=1515906&view=auto
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyUnavailableException.java
(added)
+++ hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyUnavailableException.java
Tue Aug 20 18:07:47 2013
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+
+public class ZeroCopyUnavailableException extends IOException {
+  private static final long serialVersionUID = 0L;
+
+  public ZeroCopyUnavailableException(String message) {
+    super(message);
+  }
+
+  public ZeroCopyUnavailableException(String message, Exception e) {
+    super(message, e);
+  }
+
+  public ZeroCopyUnavailableException(Exception e) {
+    super(e);
+  }
+}

Propchange: hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ZeroCopyUnavailableException.java
------------------------------------------------------------------------------
    svn:eol-style = native



Mime
View raw message