arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject [1/3] arrow git commit: ARROW-618: [Python/C++] Support timestamp+timezone conversion to pandas
Date Mon, 13 Mar 2017 20:15:57 GMT
Repository: arrow
Updated Branches:
  refs/heads/master 331be4923 -> 00df40cea


http://git-wip-us.apache.org/repos/asf/arrow/blob/00df40ce/python/src/pyarrow/io.h
----------------------------------------------------------------------
diff --git a/python/src/pyarrow/io.h b/python/src/pyarrow/io.h
index 4cb010f..a603e81 100644
--- a/python/src/pyarrow/io.h
+++ b/python/src/pyarrow/io.h
@@ -20,17 +20,17 @@
 
 #include "arrow/io/interfaces.h"
 #include "arrow/io/memory.h"
+#include "arrow/util/visibility.h"
 
 #include "pyarrow/config.h"
 
 #include "pyarrow/common.h"
-#include "pyarrow/visibility.h"
 
 namespace arrow {
+
 class MemoryPool;
-}
 
-namespace pyarrow {
+namespace py {
 
 // A common interface to a Python file-like object. Must acquire GIL before
 // calling any methods
@@ -39,31 +39,31 @@ class PythonFile {
   PythonFile(PyObject* file);
   ~PythonFile();
 
-  arrow::Status Close();
-  arrow::Status Seek(int64_t position, int whence);
-  arrow::Status Read(int64_t nbytes, PyObject** out);
-  arrow::Status Tell(int64_t* position);
-  arrow::Status Write(const uint8_t* data, int64_t nbytes);
+  Status Close();
+  Status Seek(int64_t position, int whence);
+  Status Read(int64_t nbytes, PyObject** out);
+  Status Tell(int64_t* position);
+  Status Write(const uint8_t* data, int64_t nbytes);
 
  private:
   PyObject* file_;
 };
 
-class PYARROW_EXPORT PyReadableFile : public arrow::io::ReadableFileInterface {
+class ARROW_EXPORT PyReadableFile : public io::ReadableFileInterface {
  public:
   explicit PyReadableFile(PyObject* file);
   virtual ~PyReadableFile();
 
-  arrow::Status Close() override;
+  Status Close() override;
 
-  arrow::Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override;
-  arrow::Status Read(int64_t nbytes, std::shared_ptr<arrow::Buffer>* out) override;
+  Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override;
+  Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override;
 
-  arrow::Status GetSize(int64_t* size) override;
+  Status GetSize(int64_t* size) override;
 
-  arrow::Status Seek(int64_t position) override;
+  Status Seek(int64_t position) override;
 
-  arrow::Status Tell(int64_t* position) override;
+  Status Tell(int64_t* position) override;
 
   bool supports_zero_copy() const override;
 
@@ -71,21 +71,21 @@ class PYARROW_EXPORT PyReadableFile : public arrow::io::ReadableFileInterface
{
   std::unique_ptr<PythonFile> file_;
 };
 
-class PYARROW_EXPORT PyOutputStream : public arrow::io::OutputStream {
+class ARROW_EXPORT PyOutputStream : public io::OutputStream {
  public:
   explicit PyOutputStream(PyObject* file);
   virtual ~PyOutputStream();
 
-  arrow::Status Close() override;
-  arrow::Status Tell(int64_t* position) override;
-  arrow::Status Write(const uint8_t* data, int64_t nbytes) override;
+  Status Close() override;
+  Status Tell(int64_t* position) override;
+  Status Write(const uint8_t* data, int64_t nbytes) override;
 
  private:
   std::unique_ptr<PythonFile> file_;
 };
 
 // A zero-copy reader backed by a PyBytes object
-class PYARROW_EXPORT PyBytesReader : public arrow::io::BufferReader {
+class ARROW_EXPORT PyBytesReader : public io::BufferReader {
  public:
   explicit PyBytesReader(PyObject* obj);
   virtual ~PyBytesReader();
@@ -93,6 +93,7 @@ class PYARROW_EXPORT PyBytesReader : public arrow::io::BufferReader {
 
 // TODO(wesm): seekable output files
 
-}  // namespace pyarrow
+}  // namespace py
+}  // namespace arrow
 
 #endif  // PYARROW_IO_H

http://git-wip-us.apache.org/repos/asf/arrow/blob/00df40ce/python/src/pyarrow/numpy_interop.h
----------------------------------------------------------------------
diff --git a/python/src/pyarrow/numpy_interop.h b/python/src/pyarrow/numpy_interop.h
index 6326527..57f3328 100644
--- a/python/src/pyarrow/numpy_interop.h
+++ b/python/src/pyarrow/numpy_interop.h
@@ -42,7 +42,8 @@
 #include <numpy/arrayobject.h>
 #include <numpy/ufuncobject.h>
 
-namespace pyarrow {
+namespace arrow {
+namespace py {
 
 inline int import_numpy() {
 #ifdef NUMPY_IMPORT_ARRAY
@@ -53,6 +54,7 @@ inline int import_numpy() {
   return 0;
 }
 
-}  // namespace pyarrow
+}  // namespace py
+}  // namespace arrow
 
 #endif  // PYARROW_NUMPY_INTEROP_H

http://git-wip-us.apache.org/repos/asf/arrow/blob/00df40ce/python/src/pyarrow/type_traits.h
----------------------------------------------------------------------
diff --git a/python/src/pyarrow/type_traits.h b/python/src/pyarrow/type_traits.h
new file mode 100644
index 0000000..f4604d7
--- /dev/null
+++ b/python/src/pyarrow/type_traits.h
@@ -0,0 +1,212 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <Python.h>
+
+#include <cstdint>
+
+#include "pyarrow/numpy_interop.h"
+
+#include "arrow/builder.h"
+#include "arrow/type.h"
+
+namespace arrow {
+namespace py {
+
+template <int TYPE>
+struct npy_traits {};
+
+template <>
+struct npy_traits<NPY_BOOL> {
+  typedef uint8_t value_type;
+  using TypeClass = BooleanType;
+  using BuilderClass = BooleanBuilder;
+
+  static constexpr bool supports_nulls = false;
+  static inline bool isnull(uint8_t v) { return false; }
+};
+
+#define NPY_INT_DECL(TYPE, CapType, T)               \
+  template <>                                        \
+  struct npy_traits<NPY_##TYPE> {                    \
+    typedef T value_type;                            \
+    using TypeClass = CapType##Type;                 \
+    using BuilderClass = CapType##Builder;           \
+                                                     \
+    static constexpr bool supports_nulls = false;    \
+    static inline bool isnull(T v) { return false; } \
+  };
+
+NPY_INT_DECL(INT8, Int8, int8_t);
+NPY_INT_DECL(INT16, Int16, int16_t);
+NPY_INT_DECL(INT32, Int32, int32_t);
+NPY_INT_DECL(INT64, Int64, int64_t);
+
+NPY_INT_DECL(UINT8, UInt8, uint8_t);
+NPY_INT_DECL(UINT16, UInt16, uint16_t);
+NPY_INT_DECL(UINT32, UInt32, uint32_t);
+NPY_INT_DECL(UINT64, UInt64, uint64_t);
+
+#if NPY_INT64 != NPY_LONGLONG
+NPY_INT_DECL(LONGLONG, Int64, int64_t);
+NPY_INT_DECL(ULONGLONG, UInt64, uint64_t);
+#endif
+
+template <>
+struct npy_traits<NPY_FLOAT32> {
+  typedef float value_type;
+  using TypeClass = FloatType;
+  using BuilderClass = FloatBuilder;
+
+  static constexpr bool supports_nulls = true;
+
+  static inline bool isnull(float v) { return v != v; }
+};
+
+template <>
+struct npy_traits<NPY_FLOAT64> {
+  typedef double value_type;
+  using TypeClass = DoubleType;
+  using BuilderClass = DoubleBuilder;
+
+  static constexpr bool supports_nulls = true;
+
+  static inline bool isnull(double v) { return v != v; }
+};
+
+template <>
+struct npy_traits<NPY_DATETIME> {
+  typedef int64_t value_type;
+  using TypeClass = TimestampType;
+  using BuilderClass = TimestampBuilder;
+
+  static constexpr bool supports_nulls = true;
+
+  static inline bool isnull(int64_t v) {
+    // NaT = -2**63
+    // = -0x8000000000000000
+    // = -9223372036854775808;
+    // = std::numeric_limits<int64_t>::min()
+    return v == std::numeric_limits<int64_t>::min();
+  }
+};
+
+template <>
+struct npy_traits<NPY_OBJECT> {
+  typedef PyObject* value_type;
+  static constexpr bool supports_nulls = true;
+};
+
+template <int TYPE>
+struct arrow_traits {};
+
+template <>
+struct arrow_traits<Type::BOOL> {
+  static constexpr int npy_type = NPY_BOOL;
+  static constexpr bool supports_nulls = false;
+  static constexpr bool is_boolean = true;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = false;
+};
+
+#define INT_DECL(TYPE)                                     \
+  template <>                                              \
+  struct arrow_traits<Type::TYPE> {                        \
+    static constexpr int npy_type = NPY_##TYPE;            \
+    static constexpr bool supports_nulls = false;          \
+    static constexpr double na_value = NAN;                \
+    static constexpr bool is_boolean = false;              \
+    static constexpr bool is_numeric_not_nullable = true;  \
+    static constexpr bool is_numeric_nullable = false;     \
+    typedef typename npy_traits<NPY_##TYPE>::value_type T; \
+  };
+
+INT_DECL(INT8);
+INT_DECL(INT16);
+INT_DECL(INT32);
+INT_DECL(INT64);
+INT_DECL(UINT8);
+INT_DECL(UINT16);
+INT_DECL(UINT32);
+INT_DECL(UINT64);
+
+template <>
+struct arrow_traits<Type::FLOAT> {
+  static constexpr int npy_type = NPY_FLOAT32;
+  static constexpr bool supports_nulls = true;
+  static constexpr float na_value = NAN;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = true;
+  typedef typename npy_traits<NPY_FLOAT32>::value_type T;
+};
+
+template <>
+struct arrow_traits<Type::DOUBLE> {
+  static constexpr int npy_type = NPY_FLOAT64;
+  static constexpr bool supports_nulls = true;
+  static constexpr double na_value = NAN;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = true;
+  typedef typename npy_traits<NPY_FLOAT64>::value_type T;
+};
+
+static constexpr int64_t kPandasTimestampNull = std::numeric_limits<int64_t>::min();
+
+template <>
+struct arrow_traits<Type::TIMESTAMP> {
+  static constexpr int npy_type = NPY_DATETIME;
+  static constexpr bool supports_nulls = true;
+  static constexpr int64_t na_value = kPandasTimestampNull;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = true;
+  typedef typename npy_traits<NPY_DATETIME>::value_type T;
+};
+
+template <>
+struct arrow_traits<Type::DATE> {
+  static constexpr int npy_type = NPY_DATETIME;
+  static constexpr bool supports_nulls = true;
+  static constexpr int64_t na_value = kPandasTimestampNull;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = true;
+  typedef typename npy_traits<NPY_DATETIME>::value_type T;
+};
+
+template <>
+struct arrow_traits<Type::STRING> {
+  static constexpr int npy_type = NPY_OBJECT;
+  static constexpr bool supports_nulls = true;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = false;
+};
+
+template <>
+struct arrow_traits<Type::BINARY> {
+  static constexpr int npy_type = NPY_OBJECT;
+  static constexpr bool supports_nulls = true;
+  static constexpr bool is_boolean = false;
+  static constexpr bool is_numeric_not_nullable = false;
+  static constexpr bool is_numeric_nullable = false;
+};
+
+}  // namespace py
+}  // namespace arrow

http://git-wip-us.apache.org/repos/asf/arrow/blob/00df40ce/python/src/pyarrow/util/datetime.h
----------------------------------------------------------------------
diff --git a/python/src/pyarrow/util/datetime.h b/python/src/pyarrow/util/datetime.h
index 9ffa691..f704a96 100644
--- a/python/src/pyarrow/util/datetime.h
+++ b/python/src/pyarrow/util/datetime.h
@@ -21,7 +21,8 @@
 #include <Python.h>
 #include <datetime.h>
 
-namespace pyarrow {
+namespace arrow {
+namespace py {
 
 inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
   struct tm date = {0};
@@ -35,6 +36,7 @@ inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) {
   return lrint(difftime(mktime(&date), mktime(&epoch)) * 1000);
 }
 
-}  // namespace pyarrow
+}  // namespace py
+}  // namespace arrow
 
 #endif  // PYARROW_UTIL_DATETIME_H

http://git-wip-us.apache.org/repos/asf/arrow/blob/00df40ce/python/src/pyarrow/util/test_main.cc
----------------------------------------------------------------------
diff --git a/python/src/pyarrow/util/test_main.cc b/python/src/pyarrow/util/test_main.cc
index 02e9a54..d8d1d03 100644
--- a/python/src/pyarrow/util/test_main.cc
+++ b/python/src/pyarrow/util/test_main.cc
@@ -26,7 +26,7 @@ int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
 
   Py_Initialize();
-  pyarrow::import_numpy();
+  arrow::py::import_numpy();
 
   int ret = RUN_ALL_TESTS();
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/00df40ce/python/src/pyarrow/visibility.h
----------------------------------------------------------------------
diff --git a/python/src/pyarrow/visibility.h b/python/src/pyarrow/visibility.h
deleted file mode 100644
index 9f0c13b..0000000
--- a/python/src/pyarrow/visibility.h
+++ /dev/null
@@ -1,32 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef PYARROW_VISIBILITY_H
-#define PYARROW_VISIBILITY_H
-
-#if defined(_WIN32) || defined(__CYGWIN__)
-#define PYARROW_EXPORT __declspec(dllexport)
-#else  // Not Windows
-#ifndef PYARROW_EXPORT
-#define PYARROW_EXPORT __attribute__((visibility("default")))
-#endif
-#ifndef PYARROW_NO_EXPORT
-#define PYARROW_NO_EXPORT __attribute__((visibility("hidden")))
-#endif
-#endif  // Non-Windows
-
-#endif  // PYARROW_VISIBILITY_H


Mime
View raw message