arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject arrow git commit: ARROW-239: Test reading remainder of file in HDFS with read() with no args
Date Fri, 13 Jan 2017 19:00:19 GMT
Repository: arrow
Updated Branches:
  refs/heads/master 281cb9697 -> 876ae8509


ARROW-239: Test reading remainder of file in HDFS with read() with no args

This was already fixed, but adding a unit test to verify.

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #284 from wesm/ARROW-239 and squashes the following commits:

7b3aae7 [Wes McKinney] Test reading remainder of file in HDFS with read() with no args


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/876ae850
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/876ae850
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/876ae850

Branch: refs/heads/master
Commit: 876ae8509d84b8c4dc269250b2fae47cef897054
Parents: 281cb96
Author: Wes McKinney <wes.mckinney@twosigma.com>
Authored: Fri Jan 13 14:00:12 2017 -0500
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Fri Jan 13 14:00:12 2017 -0500

----------------------------------------------------------------------
 python/pyarrow/schema.pyx         |  8 +++++---
 python/pyarrow/tests/test_hdfs.py | 12 ++++++++++++
 2 files changed, 17 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/876ae850/python/pyarrow/schema.pyx
----------------------------------------------------------------------
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index f6a1a10..a6aa9d5 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -22,6 +22,8 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
+from cython.operator cimport dereference as deref
+
 from pyarrow.compat import frombytes, tobytes
 from pyarrow.includes.libarrow cimport (CDataType, CStructType, CListType,
                                         Type_NA, Type_BOOL,
@@ -31,7 +33,7 @@ from pyarrow.includes.libarrow cimport (CDataType, CStructType, CListType,
                                         Type_UINT64, Type_INT64,
                                         Type_TIMESTAMP, Type_DATE,
                                         Type_FLOAT, Type_DOUBLE,
-                                        Type_STRING, Type_BINARY, 
+                                        Type_STRING, Type_BINARY,
                                         TimeUnit_SECOND, TimeUnit_MILLI,
                                         TimeUnit_MICRO, TimeUnit_NANO,
                                         Type, TimeUnit)
@@ -57,9 +59,9 @@ cdef class DataType:
 
     def __richcmp__(DataType self, DataType other, int op):
         if op == cpython.Py_EQ:
-            return self.type.Equals(other.sp_type)
+            return self.type.Equals(deref(other.type))
         elif op == cpython.Py_NE:
-            return not self.type.Equals(other.sp_type)
+            return not self.type.Equals(deref(other.type))
         else:
             raise TypeError('Invalid comparison')
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/876ae850/python/pyarrow/tests/test_hdfs.py
----------------------------------------------------------------------
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index 4ff5a9d..2056f7a 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -126,6 +126,18 @@ class HdfsTestCases(object):
             result = f.read(10)
             assert result == data
 
+    def test_hdfs_read_whole_file(self):
+        path = pjoin(self.tmp_path, 'read-whole-file')
+
+        data = b'foo' * 1000
+        with self.hdfs.open(path, 'wb') as f:
+            f.write(data)
+
+        with self.hdfs.open(path, 'rb') as f:
+            result = f.read()
+
+        assert result == data
+
 
 class TestLibHdfs(HdfsTestCases, unittest.TestCase):
 


Mime
View raw message