hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lium...@apache.org
Subject hadoop git commit: HADOOP-14478. Optimize NativeAzureFsInputStream for positional reads. Contributed by Rajesh Balamohan
Date Mon, 05 Jun 2017 23:01:05 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 835560983 -> 5fd9742c8


HADOOP-14478. Optimize NativeAzureFsInputStream for positional reads. Contributed by Rajesh
Balamohan


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5fd9742c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5fd9742c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5fd9742c

Branch: refs/heads/trunk
Commit: 5fd9742c83fbeae96bf0913bdcdf77fafbf15b2f
Parents: 8355609
Author: Mingliang Liu <liuml07@apache.org>
Authored: Mon Jun 5 15:56:43 2017 -0700
Committer: Mingliang Liu <liuml07@apache.org>
Committed: Mon Jun 5 15:56:43 2017 -0700

----------------------------------------------------------------------
 .../fs/azure/AzureNativeFileSystemStore.java    |  5 +---
 .../hadoop/fs/azure/NativeAzureFileSystem.java  | 31 ++++++++++++++++++--
 2 files changed, 29 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fd9742c/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index a5bb370..534919e 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -19,7 +19,6 @@
 package org.apache.hadoop.fs.azure;
 import static org.apache.hadoop.fs.azure.NativeAzureFileSystem.PATH_DELIMITER;
 
-import java.io.BufferedInputStream;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
@@ -2043,11 +2042,9 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore
{
 
         // Get blob reference and open the input buffer stream.
         CloudBlobWrapper blob = getBlobReference(key);
-      BufferedInputStream inBufStream = new BufferedInputStream(
-          openInputStream(blob));
 
         // Return a data input stream.
-        DataInputStream inDataStream = new DataInputStream(inBufStream);
+        DataInputStream inDataStream = new DataInputStream(openInputStream(blob));
         return inDataStream;
     } catch (Exception e) {
       // Re-throw as an Azure storage exception.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fd9742c/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index b61baab..b2cc4ea 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -816,6 +816,27 @@ public class NativeAzureFileSystem extends FileSystem {
       }
     }
 
+    @Override
+    public synchronized  void readFully(long position, byte[] buffer, int offset, int length)
+        throws IOException {
+      validatePositionedReadArgs(position, buffer, offset, length);
+
+      int nread = 0;
+      while (nread < length) {
+        // In case BlobInputStream is used, mark() can act as a hint to read ahead only this
+        // length instead of 4 MB boundary.
+        in.mark(length - nread);
+        int nbytes = read(position + nread,
+            buffer,
+            offset + nread,
+            length - nread);
+        if (nbytes < 0) {
+          throw new EOFException(FSExceptionMessages.EOF_IN_READ_FULLY);
+        }
+        nread += nbytes;
+      }
+    }
+
     /*
      * Reads up to len bytes of data from the input stream into an array of
      * bytes. An attempt is made to read as many as len bytes, but a smaller
@@ -886,9 +907,13 @@ public class NativeAzureFileSystem extends FileSystem {
         if (pos < 0) {
           throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
         }
-        IOUtils.closeStream(in);
-        in = store.retrieve(key);
-        this.pos = in.skip(pos);
+        if (this.pos > pos) {
+          IOUtils.closeStream(in);
+          in = store.retrieve(key);
+          this.pos = in.skip(pos);
+        } else {
+          this.pos += in.skip(pos - this.pos);
+        }
         LOG.debug("Seek to position {}. Bytes skipped {}", pos,
           this.pos);
       } catch(IOException e) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message