arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From u..@apache.org
Subject arrow git commit: ARROW-502 [C++/Python]: Logging memory pool
Date Mon, 20 Mar 2017 09:58:02 GMT
Repository: arrow
Updated Branches:
  refs/heads/master cd4544df8 -> 02bdbf48a


ARROW-502 [C++/Python]: Logging memory pool

This is a simple decorator on MemoryPool that logs it call to ``std::cout``. I can improve
it later if you need to log to other supports. Are you ok with the current logging format
?

Also, I'm not a cython expert so I hope the implementation of ``CLoggingMemoryPool`` is correct.

Author: Johan Mabille <johan.mabille@gmail.com>

Closes #395 from JohanMabille/memory_pool and squashes the following commits:

aa8ad5f [Johan Mabille] cython fix
f70e78a [Johan Mabille] python logging memory pool
9d1d144 [Johan Mabille] formatting
8f9164c [Johan Mabille] Logging memory pool


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/02bdbf48
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/02bdbf48
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/02bdbf48

Branch: refs/heads/master
Commit: 02bdbf48a483b224ebfd61cf9be69cb0807e6e50
Parents: cd4544d
Author: Johan Mabille <johan.mabille@gmail.com>
Authored: Mon Mar 20 10:57:57 2017 +0100
Committer: Uwe L. Korn <uwelk@xhochy.com>
Committed: Mon Mar 20 10:57:57 2017 +0100

----------------------------------------------------------------------
 cpp/src/arrow/memory_pool-test.cc    | 17 ++++++++++++++++
 cpp/src/arrow/memory_pool.cc         | 32 +++++++++++++++++++++++++++++++
 cpp/src/arrow/memory_pool.h          | 18 +++++++++++++++++
 python/pyarrow/includes/libarrow.pxd |  3 +++
 python/pyarrow/memory.pxd            |  5 ++++-
 python/pyarrow/memory.pyx            |  5 ++++-
 6 files changed, 78 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/02bdbf48/cpp/src/arrow/memory_pool-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/memory_pool-test.cc b/cpp/src/arrow/memory_pool-test.cc
index 6ab73fb..8a185ab 100644
--- a/cpp/src/arrow/memory_pool-test.cc
+++ b/cpp/src/arrow/memory_pool-test.cc
@@ -78,4 +78,21 @@ TEST(DefaultMemoryPoolDeathTest, MaxMemory) {
 
 #endif  // ARROW_VALGRIND
 
+TEST(LoggingMemoryPool, Logging) {
+  DefaultMemoryPool pool;
+  LoggingMemoryPool lp(&pool);
+
+  ASSERT_EQ(0, lp.max_memory());
+
+  uint8_t* data;
+  ASSERT_OK(pool.Allocate(100, &data));
+
+  uint8_t* data2;
+  ASSERT_OK(pool.Allocate(100, &data2));
+
+  pool.Free(data, 100);
+  pool.Free(data2, 100);
+
+  ASSERT_EQ(200, pool.max_memory());
+}
 }  // namespace arrow

http://git-wip-us.apache.org/repos/asf/arrow/blob/02bdbf48/cpp/src/arrow/memory_pool.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index 5a63027..cf01a02 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -22,6 +22,7 @@
 #include <mutex>
 #include <sstream>
 #include <stdlib.h>
+#include <iostream>
 
 #include "arrow/status.h"
 #include "arrow/util/logging.h"
@@ -134,4 +135,35 @@ MemoryPool* default_memory_pool() {
   return &default_memory_pool_;
 }
 
+LoggingMemoryPool::LoggingMemoryPool(MemoryPool* pool) : pool_(pool) {}
+
+Status LoggingMemoryPool::Allocate(int64_t size, uint8_t** out) {
+  Status s = pool_->Allocate(size, out);
+  std::cout << "Allocate: size = " << size << " - out = " << *out
<< std::endl;
+  return s;
+}
+
+Status LoggingMemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) {
+  Status s = pool_->Reallocate(old_size, new_size, ptr);
+  std::cout << "Reallocate: old_size = " << old_size << " - new_size =
" << new_size
+            << " - ptr = " << *ptr << std::endl;
+  return s;
+}
+
+void LoggingMemoryPool::Free(uint8_t* buffer, int64_t size) {
+  pool_->Free(buffer, size);
+  std::cout << "Free: buffer = " << buffer << " - size = " << size
<< std::endl;
+}
+
+int64_t LoggingMemoryPool::bytes_allocated() const {
+  int64_t nb_bytes = pool_->bytes_allocated();
+  std::cout << "bytes_allocated: " << nb_bytes << std::endl;
+  return nb_bytes;
+}
+
+int64_t LoggingMemoryPool::max_memory() const {
+  int64_t mem = pool_->max_memory();
+  std::cout << "max_memory: " << mem << std::endl;
+  return mem;
+}
 }  // namespace arrow

http://git-wip-us.apache.org/repos/asf/arrow/blob/02bdbf48/cpp/src/arrow/memory_pool.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/memory_pool.h b/cpp/src/arrow/memory_pool.h
index 0edfda6..90bc593 100644
--- a/cpp/src/arrow/memory_pool.h
+++ b/cpp/src/arrow/memory_pool.h
@@ -89,6 +89,24 @@ class ARROW_EXPORT DefaultMemoryPool : public MemoryPool {
   std::atomic<int64_t> max_memory_;
 };
 
+class ARROW_EXPORT LoggingMemoryPool : public MemoryPool {
+ public:
+  explicit LoggingMemoryPool(MemoryPool* pool);
+  virtual ~LoggingMemoryPool() = default;
+
+  Status Allocate(int64_t size, uint8_t** out) override;
+  Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) override;
+
+  void Free(uint8_t* buffer, int64_t size) override;
+
+  int64_t bytes_allocated() const override;
+
+  int64_t max_memory() const override;
+
+ private:
+  MemoryPool* pool_;
+};
+
 ARROW_EXPORT MemoryPool* default_memory_pool();
 
 }  // namespace arrow

http://git-wip-us.apache.org/repos/asf/arrow/blob/02bdbf48/python/pyarrow/includes/libarrow.pxd
----------------------------------------------------------------------
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index dee7fd4..705fe6b 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -104,6 +104,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
     cdef cppclass CMemoryPool" arrow::MemoryPool":
         int64_t bytes_allocated()
 
+    cdef cppclass CLoggingMemoryPool" arrow::LoggingMemoryPool"(CMemoryPool):
+        CLoggingMemoryPool(CMemoryPool*)
+        
     cdef cppclass CBuffer" arrow::Buffer":
         uint8_t* data()
         int64_t size()

http://git-wip-us.apache.org/repos/asf/arrow/blob/02bdbf48/python/pyarrow/memory.pxd
----------------------------------------------------------------------
diff --git a/python/pyarrow/memory.pxd b/python/pyarrow/memory.pxd
index 3079ccb..bb1af85 100644
--- a/python/pyarrow/memory.pxd
+++ b/python/pyarrow/memory.pxd
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pyarrow.includes.libarrow cimport CMemoryPool
+from pyarrow.includes.libarrow cimport CMemoryPool, CLoggingMemoryPool
 
 
 cdef class MemoryPool:
@@ -24,4 +24,7 @@ cdef class MemoryPool:
 
     cdef init(self, CMemoryPool* pool)
 
+cdef class LoggingMemoryPool(MemoryPool):
+    pass
+
 cdef CMemoryPool* maybe_unbox_memory_pool(MemoryPool memory_pool)

http://git-wip-us.apache.org/repos/asf/arrow/blob/02bdbf48/python/pyarrow/memory.pyx
----------------------------------------------------------------------
diff --git a/python/pyarrow/memory.pyx b/python/pyarrow/memory.pyx
index 18a6de4..98dbf66 100644
--- a/python/pyarrow/memory.pyx
+++ b/python/pyarrow/memory.pyx
@@ -19,7 +19,7 @@
 # distutils: language = c++
 # cython: embedsignature = True
 
-from pyarrow.includes.libarrow cimport CMemoryPool
+from pyarrow.includes.libarrow cimport CMemoryPool, CLoggingMemoryPool
 from pyarrow.includes.pyarrow cimport set_default_memory_pool, get_memory_pool
 
 cdef class MemoryPool:
@@ -35,6 +35,9 @@ cdef CMemoryPool* maybe_unbox_memory_pool(MemoryPool memory_pool):
     else:
         return memory_pool.pool
 
+cdef class LoggingMemoryPool(MemoryPool):
+    pass
+
 def default_pool():
     cdef: 
         MemoryPool pool = MemoryPool()


Mime
View raw message