arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject arrow git commit: ARROW-450: Fixes for PARQUET-818
Date Mon, 02 Jan 2017 07:37:46 GMT
Repository: arrow
Updated Branches:
  refs/heads/master e15c6a0b3 -> e8b6231b2


ARROW-450: Fixes for PARQUET-818

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #263 from xhochy/ARROW-450 and squashes the following commits:

287015a [Uwe L. Korn] ARROW-450: Fixes for PARQUET-818


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/e8b6231b
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/e8b6231b
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/e8b6231b

Branch: refs/heads/master
Commit: e8b6231b29f59b2978b78a33eff73697d537c5dd
Parents: e15c6a0
Author: Uwe L. Korn <uwelk@xhochy.com>
Authored: Mon Jan 2 02:37:37 2017 -0500
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Mon Jan 2 02:37:37 2017 -0500

----------------------------------------------------------------------
 python/pyarrow/includes/parquet.pxd | 19 ++-----------------
 python/pyarrow/parquet.pyx          | 15 ++++++---------
 2 files changed, 8 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/e8b6231b/python/pyarrow/includes/parquet.pxd
----------------------------------------------------------------------
diff --git a/python/pyarrow/includes/parquet.pxd b/python/pyarrow/includes/parquet.pxd
index cb791e1..b4d127c 100644
--- a/python/pyarrow/includes/parquet.pxd
+++ b/python/pyarrow/includes/parquet.pxd
@@ -120,24 +120,9 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:
             shared_ptr[WriterProperties] build()
 
 
-cdef extern from "parquet/arrow/io.h" namespace "parquet::arrow" nogil:
-    cdef cppclass ParquetAllocator:
-        ParquetAllocator()
-        ParquetAllocator(MemoryPool* pool)
-        MemoryPool* pool()
-        void set_pool(MemoryPool* pool)
-
-    cdef cppclass ParquetReadSource:
-        ParquetReadSource(ParquetAllocator* allocator)
-        Open(const shared_ptr[ReadableFileInterface]& file)
-
-    cdef cppclass ParquetWriteSink:
-        ParquetWriteSink(const shared_ptr[OutputStream]& file)
-
-
 cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
     CStatus OpenFile(const shared_ptr[ReadableFileInterface]& file,
-                     ParquetAllocator* allocator,
+                     MemoryPool* allocator,
                      unique_ptr[FileReader]* reader)
 
     cdef cppclass FileReader:
@@ -157,6 +142,6 @@ cdef extern from "parquet/arrow/schema.h" namespace "parquet::arrow" nogil:
 cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil:
     cdef CStatus WriteFlatTable(
         const CTable* table, MemoryPool* pool,
-        const shared_ptr[ParquetWriteSink]& sink,
+        const shared_ptr[OutputStream]& sink,
         int64_t chunk_size,
         const shared_ptr[WriterProperties]& properties)

http://git-wip-us.apache.org/repos/asf/arrow/blob/e8b6231b/python/pyarrow/parquet.pyx
----------------------------------------------------------------------
diff --git a/python/pyarrow/parquet.pyx b/python/pyarrow/parquet.pyx
index 043ccf1..7379456 100644
--- a/python/pyarrow/parquet.pyx
+++ b/python/pyarrow/parquet.pyx
@@ -42,12 +42,12 @@ __all__ = [
 
 cdef class ParquetReader:
     cdef:
-        ParquetAllocator allocator
+        MemoryPool* allocator
         unique_ptr[FileReader] reader
         column_idx_map
 
     def __cinit__(self):
-        self.allocator.set_pool(default_memory_pool())
+        self.allocator = default_memory_pool()
 
     def open(self, source):
         self._open(source)
@@ -69,7 +69,7 @@ cdef class ParquetReader:
                                ParquetFileReader.OpenFile(path)))
         else:
             get_reader(source, &rd_handle)
-            check_status(OpenFile(rd_handle, &self.allocator, &self.reader))
+            check_status(OpenFile(rd_handle, self.allocator, &self.reader))
 
     def read_all(self):
         cdef:
@@ -174,10 +174,8 @@ def write_table(table, sink, chunk_size=None, version=None,
     """
     cdef Table table_ = table
     cdef CTable* ctable_ = table_.table
-    cdef shared_ptr[ParquetWriteSink] sink_
-
     cdef shared_ptr[FileOutputStream] filesink_
-    cdef shared_ptr[OutputStream] general_sink
+    cdef shared_ptr[OutputStream] sink_
 
     cdef WriterProperties.Builder properties_builder
     cdef int64_t chunk_size_ = 0
@@ -237,10 +235,9 @@ def write_table(table, sink, chunk_size=None, version=None,
 
     if isinstance(sink, six.string_types):
         check_status(FileOutputStream.Open(tobytes(sink), &filesink_))
-        sink_.reset(new ParquetWriteSink(<shared_ptr[OutputStream]>filesink_))
+        sink_ = <shared_ptr[OutputStream]>filesink_
     else:
-        get_writer(sink, &general_sink)
-        sink_.reset(new ParquetWriteSink(general_sink))
+        get_writer(sink, &sink_)
 
     with nogil:
         check_status(WriteFlatTable(ctable_, default_memory_pool(), sink_,


Mime
View raw message