arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject [arrow] branch master updated: ARROW-1854: [Python] Use pickle to serialize numpy arrays of objects.
Date Wed, 29 Nov 2017 01:06:22 GMT
This is an automated email from the ASF dual-hosted git repository.

wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git


The following commit(s) were added to refs/heads/master by this push:
     new 0d6f5bf  ARROW-1854: [Python] Use pickle to serialize numpy arrays of objects.
0d6f5bf is described below

commit 0d6f5bf5fab44a83669cf9d9af5c023f15793dc0
Author: Wes McKinney <wes.mckinney@twosigma.com>
AuthorDate: Tue Nov 28 20:06:18 2017 -0500

    ARROW-1854: [Python] Use pickle to serialize numpy arrays of objects.
    
    **Just posting this for discussion.** See the preceding discussion on https://issues.apache.org/jira/browse/ARROW-1854.
    
    I think the ideal way to solve this would actually be to improve our handling of lists,
which should be possible given that pickle seems to outperform us by 6x according to the benchmarks
in https://issues.apache.org/jira/browse/ARROW-1854.
    
    Note that the implementation in this PR will not handle numpy arrays of user-defined classes
because it will not fall back to cloudpickle when needed.
    
    cc @pcmoritz @wesm
    
    Author: Wes McKinney <wes.mckinney@twosigma.com>
    Author: Robert Nishihara <robertnishihara@gmail.com>
    
    Closes #1360 from robertnishihara/numpyobject and squashes the following commits:
    
    c37a0a08 [Wes McKinney] Fix flake
    51915032 [Wes McKinney] Fix post rebase
    43f2c805 [Wes McKinney] Add SerializationContext.clone method. Add pandas_serialization_context
member that uses pickle for NumPy arrays with unsupported tensor types
    c9440231 [Wes McKinney] Use pickle.HIGHEST_PROTOCOL, convert to Buffer then memoryview
for more memory-efficient transport
    cf719c3f [Robert Nishihara] Use pickle to serialize numpy arrays of objects.
---
 python/pyarrow/__init__.py                 |  1 +
 python/pyarrow/serialization.pxi           | 16 +++++++++++
 python/pyarrow/serialization.py            | 46 +++++++++++++++++++++---------
 python/pyarrow/tests/test_serialization.py |  7 +++--
 4 files changed, 53 insertions(+), 17 deletions(-)

diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index bd31b21..a245fe6 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -125,6 +125,7 @@ from pyarrow.ipc import (Message, MessageReader,
 localfs = LocalFileSystem.get_instance()
 
 from pyarrow.serialization import (_default_serialization_context,
+                                   pandas_serialization_context,
                                    register_default_serialization_handlers)
 
 import pyarrow.types as types
diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi
index bb266b2..faf164b 100644
--- a/python/pyarrow/serialization.pxi
+++ b/python/pyarrow/serialization.pxi
@@ -57,6 +57,22 @@ cdef class SerializationContext:
         self.custom_serializers = dict()
         self.custom_deserializers = dict()
 
+    def clone(self):
+        """
+        Return copy of this SerializationContext
+
+        Returns
+        -------
+        clone : SerializationContext
+        """
+        result = SerializationContext()
+        result.type_to_type_id = self.type_to_type_id.copy()
+        result.whitelisted_types = self.whitelisted_types.copy()
+        result.custom_serializers = self.custom_serializers.copy()
+        result.custom_deserializers = self.custom_deserializers.copy()
+
+        return result
+
     def register_type(self, type_, type_id,
                       custom_serializer=None, custom_deserializer=None):
         """EXPERIMENTAL: Add type to the list of types we can serialize.
diff --git a/python/pyarrow/serialization.py b/python/pyarrow/serialization.py
index ab25b63..08e6cce 100644
--- a/python/pyarrow/serialization.py
+++ b/python/pyarrow/serialization.py
@@ -22,7 +22,7 @@ import pickle
 import numpy as np
 
 from pyarrow import serialize_pandas, deserialize_pandas
-from pyarrow.lib import _default_serialization_context
+from pyarrow.lib import _default_serialization_context, frombuffer
 
 try:
     import cloudpickle
@@ -30,6 +30,28 @@ except ImportError:
     cloudpickle = pickle
 
 
+# ----------------------------------------------------------------------
+# Set up serialization for numpy with dtype object (primitive types are
+# handled efficiently with Arrow's Tensor facilities, see
+# python_to_arrow.cc)
+
+def _serialize_numpy_array_list(obj):
+    return obj.tolist(), obj.dtype.str
+
+
+def _deserialize_numpy_array_list(data):
+    return np.array(data[0], dtype=np.dtype(data[1]))
+
+
+def _serialize_numpy_array_pickle(obj):
+    pickled = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
+    return frombuffer(pickled)
+
+
+def _deserialize_numpy_array_pickle(data):
+    return pickle.loads(memoryview(data))
+
+
 def register_default_serialization_handlers(serialization_context):
 
     # ----------------------------------------------------------------------
@@ -80,21 +102,10 @@ def register_default_serialization_handlers(serialization_context):
                                         custom_serializer=cloudpickle.dumps,
                                         custom_deserializer=cloudpickle.loads)
 
-    # ----------------------------------------------------------------------
-    # Set up serialization for numpy with dtype object (primitive types are
-    # handled efficiently with Arrow's Tensor facilities, see
-    # python_to_arrow.cc)
-
-    def _serialize_numpy_array(obj):
-        return obj.tolist(), obj.dtype.str
-
-    def _deserialize_numpy_array(data):
-        return np.array(data[0], dtype=np.dtype(data[1]))
-
     serialization_context.register_type(
         np.ndarray, 'np.array',
-        custom_serializer=_serialize_numpy_array,
-        custom_deserializer=_deserialize_numpy_array)
+        custom_serializer=_serialize_numpy_array_list,
+        custom_deserializer=_deserialize_numpy_array_list)
 
     # ----------------------------------------------------------------------
     # Set up serialization for pandas Series and DataFrame
@@ -153,3 +164,10 @@ def register_default_serialization_handlers(serialization_context):
 
 
 register_default_serialization_handlers(_default_serialization_context)
+
+pandas_serialization_context = _default_serialization_context.clone()
+
+pandas_serialization_context.register_type(
+    np.ndarray, 'np.array',
+    custom_serializer=_serialize_numpy_array_pickle,
+    custom_deserializer=_deserialize_numpy_array_pickle)
diff --git a/python/pyarrow/tests/test_serialization.py b/python/pyarrow/tests/test_serialization.py
index d06beea..6d85621 100644
--- a/python/pyarrow/tests/test_serialization.py
+++ b/python/pyarrow/tests/test_serialization.py
@@ -212,11 +212,11 @@ def make_serialization_context():
 serialization_context = make_serialization_context()
 
 
-def serialization_roundtrip(value, f):
+def serialization_roundtrip(value, f, ctx=serialization_context):
     f.seek(0)
-    pa.serialize_to(value, f, serialization_context)
+    pa.serialize_to(value, f, ctx)
     f.seek(0)
-    result = pa.deserialize_from(f, None, serialization_context)
+    result = pa.deserialize_from(f, None, ctx)
     assert_equal(value, result)
 
     _check_component_roundtrip(value)
@@ -249,6 +249,7 @@ def test_primitive_serialization(large_memory_map):
     with pa.memory_map(large_memory_map, mode="r+") as mmap:
         for obj in PRIMITIVE_OBJECTS:
             serialization_roundtrip(obj, mmap)
+            serialization_roundtrip(obj, mmap, pa.pandas_serialization_context)
 
 
 def test_serialize_to_buffer():

-- 
To stop receiving notification emails like this one, please contact
['"commits@arrow.apache.org" <commits@arrow.apache.org>'].

Mime
View raw message