arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject arrow git commit: ARROW-574: Python: Add support for nested Python lists in Pandas conversion
Date Mon, 13 Mar 2017 03:18:29 GMT
Repository: arrow
Updated Branches:
  refs/heads/master 492b3d439 -> 2cf36ef2d


ARROW-574: Python: Add support for nested Python lists in Pandas conversion

Author: Uwe L. Korn <uwelk@xhochy.com>

Closes #364 from xhochy/ARROW-574 and squashes the following commits:

3ef02ae [Uwe L. Korn] ARROW-574: Python: Add support for nested Python lists in Pandas conversion


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/2cf36ef2
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/2cf36ef2
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/2cf36ef2

Branch: refs/heads/master
Commit: 2cf36ef2d6aea6a5ddf32c900d33db40d728bcd9
Parents: 492b3d4
Author: Uwe L. Korn <uwelk@xhochy.com>
Authored: Sun Mar 12 23:18:23 2017 -0400
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Sun Mar 12 23:18:23 2017 -0400

----------------------------------------------------------------------
 python/pyarrow/tests/pandas_examples.py     | 40 ++++++++++++++++++++++++
 python/pyarrow/tests/test_convert_pandas.py | 14 +++++++--
 python/pyarrow/tests/test_parquet.py        | 17 ++++++++--
 python/src/pyarrow/adapters/builtin.cc      | 27 +++++++++-------
 python/src/pyarrow/adapters/builtin.h       |  7 +++++
 python/src/pyarrow/adapters/pandas.cc       | 25 +++++++++++++--
 6 files changed, 112 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/2cf36ef2/python/pyarrow/tests/pandas_examples.py
----------------------------------------------------------------------
diff --git a/python/pyarrow/tests/pandas_examples.py b/python/pyarrow/tests/pandas_examples.py
index 63af423..c9343fc 100644
--- a/python/pyarrow/tests/pandas_examples.py
+++ b/python/pyarrow/tests/pandas_examples.py
@@ -76,3 +76,43 @@ def dataframe_with_arrays():
     schema = pa.Schema.from_fields(fields)
 
     return df, schema
+
+def dataframe_with_lists():
+    """
+    Dataframe with list columns of every possible primtive type.
+
+    Returns
+    -------
+    df: pandas.DataFrame
+    schema: pyarrow.Schema
+        Arrow schema definition that is in line with the constructed df.
+    """
+    arrays = OrderedDict()
+    fields = []
+
+    fields.append(pa.field('int64', pa.list_(pa.int64())))
+    arrays['int64'] = [
+        [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
+        [0, 1, 2, 3, 4],
+        None,
+        [0]
+    ]
+    fields.append(pa.field('double', pa.list_(pa.double())))
+    arrays['double'] = [
+        [0., 1., 2., 3., 4., 5., 6., 7., 8., 9.],
+        [0., 1., 2., 3., 4.],
+        None,
+        [0.]
+    ]
+    fields.append(pa.field('str_list', pa.list_(pa.string())))
+    arrays['str_list'] = [
+        [u"1", u"รค"],
+        None,
+        [u"1"],
+        [u"1", u"2", u"3"]
+    ]
+
+    df = pd.DataFrame(arrays)
+    schema = pa.Schema.from_fields(fields)
+
+    return df, schema

http://git-wip-us.apache.org/repos/asf/arrow/blob/2cf36ef2/python/pyarrow/tests/test_convert_pandas.py
----------------------------------------------------------------------
diff --git a/python/pyarrow/tests/test_convert_pandas.py b/python/pyarrow/tests/test_convert_pandas.py
index 953fa2c..a79bb23 100644
--- a/python/pyarrow/tests/test_convert_pandas.py
+++ b/python/pyarrow/tests/test_convert_pandas.py
@@ -30,7 +30,7 @@ import pandas.util.testing as tm
 from pyarrow.compat import u
 import pyarrow as A
 
-from .pandas_examples import dataframe_with_arrays
+from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
 
 
 def _alltypes_example(size=100):
@@ -333,7 +333,7 @@ class TestPandasConversion(unittest.TestCase):
         expected['date'] = pd.to_datetime(df['date'])
         tm.assert_frame_equal(result, expected)
 
-    def test_column_of_lists(self):
+    def test_column_of_arrays(self):
         df, schema = dataframe_with_arrays()
         self._check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
         table = A.Table.from_pandas(df, schema=schema)
@@ -343,6 +343,16 @@ class TestPandasConversion(unittest.TestCase):
             field = schema.field_by_name(column)
             self._check_array_roundtrip(df[column], field=field)
 
+    def test_column_of_lists(self):
+        df, schema = dataframe_with_lists()
+        self._check_pandas_roundtrip(df, schema=schema, expected_schema=schema)
+        table = A.Table.from_pandas(df, schema=schema)
+        assert table.schema.equals(schema)
+
+        for column in df.columns:
+            field = schema.field_by_name(column)
+            self._check_array_roundtrip(df[column], field=field)
+
     def test_threaded_conversion(self):
         df = _alltypes_example()
         self._check_pandas_roundtrip(df, nthreads=2,

http://git-wip-us.apache.org/repos/asf/arrow/blob/2cf36ef2/python/pyarrow/tests/test_parquet.py
----------------------------------------------------------------------
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 96f2d15..c72ff9e 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -23,7 +23,7 @@ import pytest
 from pyarrow.compat import guid
 import pyarrow as pa
 import pyarrow.io as paio
-from .pandas_examples import dataframe_with_arrays
+from .pandas_examples import dataframe_with_arrays, dataframe_with_lists
 
 import numpy as np
 import pandas as pd
@@ -322,7 +322,7 @@ def test_compare_schemas():
 
 
 @parquet
-def test_column_of_lists(tmpdir):
+def test_column_of_arrays(tmpdir):
     df, schema = dataframe_with_arrays()
 
     filename = tmpdir.join('pandas_rountrip.parquet')
@@ -335,6 +335,19 @@ def test_column_of_lists(tmpdir):
 
 
 @parquet
+def test_column_of_lists(tmpdir):
+    df, schema = dataframe_with_lists()
+
+    filename = tmpdir.join('pandas_rountrip.parquet')
+    arrow_table = pa.Table.from_pandas(df, timestamps_to_ms=True,
+                                       schema=schema)
+    pq.write_table(arrow_table, filename.strpath, version="2.0")
+    table_read = pq.read_table(filename.strpath)
+    df_read = table_read.to_pandas()
+    pdt.assert_frame_equal(df, df_read)
+
+
+@parquet
 def test_multithreaded_read():
     df = alltypes_sample(size=10000)
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/2cf36ef2/python/src/pyarrow/adapters/builtin.cc
----------------------------------------------------------------------
diff --git a/python/src/pyarrow/adapters/builtin.cc b/python/src/pyarrow/adapters/builtin.cc
index c125cc0..4f7b2cb 100644
--- a/python/src/pyarrow/adapters/builtin.cc
+++ b/python/src/pyarrow/adapters/builtin.cc
@@ -206,8 +206,7 @@ class SeqVisitor {
 };
 
 // Non-exhaustive type inference
-static Status InferArrowType(
-    PyObject* obj, int64_t* size, std::shared_ptr<DataType>* out_type) {
+Status InferArrowType(PyObject* obj, int64_t* size, std::shared_ptr<DataType>* out_type)
{
   *size = PySequence_Size(obj);
   if (PyErr_Occurred()) {
     // Not a sequence
@@ -496,6 +495,19 @@ Status ListConverter::Init(const std::shared_ptr<ArrayBuilder>&
builder) {
   return Status::OK();
 }
 
+Status AppendPySequence(PyObject* obj, const std::shared_ptr<DataType>& type,
+    const std::shared_ptr<ArrayBuilder>& builder) {
+  std::shared_ptr<SeqConverter> converter = GetConverter(type);
+  if (converter == nullptr) {
+    std::stringstream ss;
+    ss << "No type converter implemented for " << type->ToString();
+    return Status::NotImplemented(ss.str());
+  }
+  converter->Init(builder);
+
+  return converter->AppendData(obj);
+}
+
 Status ConvertPySequence(
     PyObject* obj, MemoryPool* pool, std::shared_ptr<arrow::Array>* out) {
   std::shared_ptr<DataType> type;
@@ -509,19 +521,10 @@ Status ConvertPySequence(
     return Status::OK();
   }
 
-  std::shared_ptr<SeqConverter> converter = GetConverter(type);
-  if (converter == nullptr) {
-    std::stringstream ss;
-    ss << "No type converter implemented for " << type->ToString();
-    return Status::NotImplemented(ss.str());
-  }
-
   // Give the sequence converter an array builder
   std::shared_ptr<ArrayBuilder> builder;
   RETURN_NOT_OK(arrow::MakeBuilder(pool, type, &builder));
-  converter->Init(builder);
-
-  RETURN_NOT_OK(converter->AppendData(obj));
+  RETURN_NOT_OK(AppendPySequence(obj, type, builder));
 
   return builder->Finish(out);
 }

http://git-wip-us.apache.org/repos/asf/arrow/blob/2cf36ef2/python/src/pyarrow/adapters/builtin.h
----------------------------------------------------------------------
diff --git a/python/src/pyarrow/adapters/builtin.h b/python/src/pyarrow/adapters/builtin.h
index 667298e..0c863a5 100644
--- a/python/src/pyarrow/adapters/builtin.h
+++ b/python/src/pyarrow/adapters/builtin.h
@@ -37,6 +37,13 @@ class Status;
 
 namespace pyarrow {
 
+PYARROW_EXPORT arrow::Status InferArrowType(
+    PyObject* obj, int64_t* size, std::shared_ptr<arrow::DataType>* out_type);
+
+PYARROW_EXPORT arrow::Status AppendPySequence(PyObject* obj,
+    const std::shared_ptr<arrow::DataType>& type,
+    const std::shared_ptr<arrow::ArrayBuilder>& builder);
+
 PYARROW_EXPORT
 arrow::Status ConvertPySequence(
     PyObject* obj, arrow::MemoryPool* pool, std::shared_ptr<arrow::Array>* out);

http://git-wip-us.apache.org/repos/asf/arrow/blob/2cf36ef2/python/src/pyarrow/adapters/pandas.cc
----------------------------------------------------------------------
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index eb3ab49..40079b4 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -19,6 +19,7 @@
 
 #include <Python.h>
 
+#include "pyarrow/adapters/builtin.h"
 #include "pyarrow/adapters/pandas.h"
 #include "pyarrow/numpy_interop.h"
 
@@ -1661,6 +1662,7 @@ inline Status ArrowSerializer<TYPE>::ConvertTypedLists(
   typedef npy_traits<ITEM_TYPE> traits;
   typedef typename traits::value_type T;
   typedef typename traits::BuilderClass BuilderT;
+  PyAcquireGIL lock;
 
   auto value_builder = std::make_shared<BuilderT>(pool_, field->type);
   ListBuilder list_builder(pool_, value_builder);
@@ -1688,7 +1690,16 @@ inline Status ArrowSerializer<TYPE>::ConvertTypedLists(
         RETURN_NOT_OK(value_builder->Append(data, size));
       }
     } else if (PyList_Check(objects[i])) {
-      return Status::TypeError("Python lists are not yet supported");
+      int64_t size;
+      std::shared_ptr<arrow::DataType> type;
+      RETURN_NOT_OK(list_builder.Append(true));
+      RETURN_NOT_OK(InferArrowType(objects[i], &size, &type));
+      if (type->type != field->type->type) {
+        std::stringstream ss;
+        ss << type->ToString() << " cannot be converted to " << field->type->ToString();
+        return Status::TypeError(ss.str());
+      }
+      RETURN_NOT_OK(AppendPySequence(objects[i], field->type, value_builder));
     } else {
       return Status::TypeError("Unsupported Python type for list items");
     }
@@ -1702,6 +1713,7 @@ inline Status
 ArrowSerializer<NPY_OBJECT>::ConvertTypedLists<NPY_OBJECT, ::arrow::StringType>(
     const std::shared_ptr<Field>& field, std::shared_ptr<Array>* out) {
   // TODO: If there are bytes involed, convert to Binary representation
+  PyAcquireGIL lock;
   bool have_bytes = false;
 
   auto value_builder = std::make_shared<arrow::StringBuilder>(pool_);
@@ -1721,7 +1733,16 @@ ArrowSerializer<NPY_OBJECT>::ConvertTypedLists<NPY_OBJECT,
::arrow::StringType>(
       auto data = reinterpret_cast<PyObject**>(PyArray_DATA(numpy_array));
       RETURN_NOT_OK(AppendObjectStrings(*value_builder.get(), data, size, &have_bytes));
     } else if (PyList_Check(objects[i])) {
-      return Status::TypeError("Python lists are not yet supported");
+      int64_t size;
+      std::shared_ptr<arrow::DataType> type;
+      RETURN_NOT_OK(list_builder.Append(true));
+      RETURN_NOT_OK(InferArrowType(objects[i], &size, &type));
+      if (type->type != Type::STRING) {
+        std::stringstream ss;
+        ss << type->ToString() << " cannot be converted to STRING.";
+        return Status::TypeError(ss.str());
+      }
+      RETURN_NOT_OK(AppendPySequence(objects[i], type, value_builder));
     } else {
       return Status::TypeError("Unsupported Python type for list items");
     }


Mime
View raw message