beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lc...@apache.org
Subject [1/3] beam git commit: Fall back to inputStream.read(byteBuffer) in case of off-heap byteBuffer use guava ByteStreams in place of manual byte skip in UTests
Date Fri, 15 Sep 2017 15:08:35 GMT
Repository: beam
Updated Branches:
  refs/heads/master c09886a4d -> 082f70c93


Fall back to inputStream.read(byteBuffer) in case of off-heap byteBuffer
use guava ByteStreams in place of manual byte skip in UTests


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/9b652efa
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/9b652efa
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/9b652efa

Branch: refs/heads/master
Commit: 9b652efab1b7b489588a6a8e16f9750124e9e4a2
Parents: c329752
Author: Etienne Chauchot <echauchot@gmail.com>
Authored: Fri Sep 15 11:44:04 2017 +0200
Committer: Luke Cwik <lcwik@google.com>
Committed: Fri Sep 15 08:04:11 2017 -0700

----------------------------------------------------------------------
 .../java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java     | 6 +++++-
 .../java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java | 4 +---
 2 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/9b652efa/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java
b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java
index ec71a53..f7a1a49 100644
--- a/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java
+++ b/sdks/java/io/hadoop-file-system/src/main/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystem.java
@@ -194,11 +194,15 @@ class HadoopFileSystem extends FileSystem<HadoopResourceId> {
       // We avoid using the ByteBuffer based read for Hadoop because some FSDataInputStream
       // implementations are not ByteBufferReadable,
       // See https://issues.apache.org/jira/browse/HADOOP-14603
-      if (dst.hasArray()){
+      if (dst.hasArray()) {
         // does the same as inputStream.read(dst):
         // stores up to dst.remaining() bytes into dst.array() starting at dst.position().
         // But dst can have an offset with its backing array hence the + dst.arrayOffset()
         read = inputStream.read(dst.array(), dst.position() + dst.arrayOffset(), dst.remaining());
+      } else {
+        // TODO: Add support for off heap ByteBuffers in case the underlying FSDataInputStream
+        // does not support reading from a ByteBuffer.
+        read = inputStream.read(dst);
       }
       if (read > 0) {
         dst.position(dst.position() + read);

http://git-wip-us.apache.org/repos/asf/beam/blob/9b652efa/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java
b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java
index 18d78c9..538141f 100644
--- a/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java
+++ b/sdks/java/io/hadoop-file-system/src/test/java/org/apache/beam/sdk/io/hdfs/HadoopFileSystemTest.java
@@ -293,9 +293,7 @@ public class HadoopFileSystemTest {
     try (ReadableByteChannel channel = fileSystem.open(testPath(relativePath))) {
       InputStream inputStream = Channels.newInputStream(channel);
       if (bytesToSkip > 0) {
-        long bytesSkip = inputStream.skip(bytesToSkip);
-        // might skip less than bytesToSkip
-        assertEquals(bytesSkip, bytesToSkip);
+        ByteStreams.skipFully(inputStream, bytesToSkip);
       }
       return ByteStreams.toByteArray(inputStream);
     }


Mime
View raw message