Repository: arrow Updated Branches: refs/heads/master cd4544df8 -> 02bdbf48a
ARROW-502 [C++/Python]: Logging memory pool This is a simple decorator on MemoryPool that logs it call to ``std::cout``. I can improve it later if you need to log to other supports. Are you ok with the current logging format ? Also, I'm not a cython expert so I hope the implementation of ``CLoggingMemoryPool`` is correct. Author: Johan Mabille <johan.mabi...@gmail.com> Closes #395 from JohanMabille/memory_pool and squashes the following commits: aa8ad5f [Johan Mabille] cython fix f70e78a [Johan Mabille] python logging memory pool 9d1d144 [Johan Mabille] formatting 8f9164c [Johan Mabille] Logging memory pool Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/02bdbf48 Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/02bdbf48 Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/02bdbf48 Branch: refs/heads/master Commit: 02bdbf48a483b224ebfd61cf9be69cb0807e6e50 Parents: cd4544d Author: Johan Mabille <johan.mabi...@gmail.com> Authored: Mon Mar 20 10:57:57 2017 +0100 Committer: Uwe L. Korn <uw...@xhochy.com> Committed: Mon Mar 20 10:57:57 2017 +0100 ---------------------------------------------------------------------- cpp/src/arrow/memory_pool-test.cc | 17 ++++++++++++++++ cpp/src/arrow/memory_pool.cc | 32 +++++++++++++++++++++++++++++++ cpp/src/arrow/memory_pool.h | 18 +++++++++++++++++ python/pyarrow/includes/libarrow.pxd | 3 +++ python/pyarrow/memory.pxd | 5 ++++- python/pyarrow/memory.pyx | 5 ++++- 6 files changed, 78 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/02bdbf48/cpp/src/arrow/memory_pool-test.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/memory_pool-test.cc b/cpp/src/arrow/memory_pool-test.cc index 6ab73fb..8a185ab 100644 --- a/cpp/src/arrow/memory_pool-test.cc +++ b/cpp/src/arrow/memory_pool-test.cc @@ -78,4 +78,21 @@ TEST(DefaultMemoryPoolDeathTest, MaxMemory) { #endif // ARROW_VALGRIND +TEST(LoggingMemoryPool, Logging) { + DefaultMemoryPool pool; + LoggingMemoryPool lp(&pool); + + ASSERT_EQ(0, lp.max_memory()); + + uint8_t* data; + ASSERT_OK(pool.Allocate(100, &data)); + + uint8_t* data2; + ASSERT_OK(pool.Allocate(100, &data2)); + + pool.Free(data, 100); + pool.Free(data2, 100); + + ASSERT_EQ(200, pool.max_memory()); +} } // namespace arrow http://git-wip-us.apache.org/repos/asf/arrow/blob/02bdbf48/cpp/src/arrow/memory_pool.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc index 5a63027..cf01a02 100644 --- a/cpp/src/arrow/memory_pool.cc +++ b/cpp/src/arrow/memory_pool.cc @@ -22,6 +22,7 @@ #include <mutex> #include <sstream> #include <stdlib.h> +#include <iostream> #include "arrow/status.h" #include "arrow/util/logging.h" @@ -134,4 +135,35 @@ MemoryPool* default_memory_pool() { return &default_memory_pool_; } +LoggingMemoryPool::LoggingMemoryPool(MemoryPool* pool) : pool_(pool) {} + +Status LoggingMemoryPool::Allocate(int64_t size, uint8_t** out) { + Status s = pool_->Allocate(size, out); + std::cout << "Allocate: size = " << size << " - out = " << *out << std::endl; + return s; +} + +Status LoggingMemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) { + Status s = pool_->Reallocate(old_size, new_size, ptr); + std::cout << "Reallocate: old_size = " << old_size << " - new_size = " << new_size + << " - ptr = " << *ptr << std::endl; + return s; +} + +void LoggingMemoryPool::Free(uint8_t* buffer, int64_t size) { + pool_->Free(buffer, size); + std::cout << "Free: buffer = " << buffer << " - size = " << size << std::endl; +} + +int64_t LoggingMemoryPool::bytes_allocated() const { + int64_t nb_bytes = pool_->bytes_allocated(); + std::cout << "bytes_allocated: " << nb_bytes << std::endl; + return nb_bytes; +} + +int64_t LoggingMemoryPool::max_memory() const { + int64_t mem = pool_->max_memory(); + std::cout << "max_memory: " << mem << std::endl; + return mem; +} } // namespace arrow http://git-wip-us.apache.org/repos/asf/arrow/blob/02bdbf48/cpp/src/arrow/memory_pool.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/memory_pool.h b/cpp/src/arrow/memory_pool.h index 0edfda6..90bc593 100644 --- a/cpp/src/arrow/memory_pool.h +++ b/cpp/src/arrow/memory_pool.h @@ -89,6 +89,24 @@ class ARROW_EXPORT DefaultMemoryPool : public MemoryPool { std::atomic<int64_t> max_memory_; }; +class ARROW_EXPORT LoggingMemoryPool : public MemoryPool { + public: + explicit LoggingMemoryPool(MemoryPool* pool); + virtual ~LoggingMemoryPool() = default; + + Status Allocate(int64_t size, uint8_t** out) override; + Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) override; + + void Free(uint8_t* buffer, int64_t size) override; + + int64_t bytes_allocated() const override; + + int64_t max_memory() const override; + + private: + MemoryPool* pool_; +}; + ARROW_EXPORT MemoryPool* default_memory_pool(); } // namespace arrow http://git-wip-us.apache.org/repos/asf/arrow/blob/02bdbf48/python/pyarrow/includes/libarrow.pxd ---------------------------------------------------------------------- diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index dee7fd4..705fe6b 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -104,6 +104,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: cdef cppclass CMemoryPool" arrow::MemoryPool": int64_t bytes_allocated() + cdef cppclass CLoggingMemoryPool" arrow::LoggingMemoryPool"(CMemoryPool): + CLoggingMemoryPool(CMemoryPool*) + cdef cppclass CBuffer" arrow::Buffer": uint8_t* data() int64_t size() http://git-wip-us.apache.org/repos/asf/arrow/blob/02bdbf48/python/pyarrow/memory.pxd ---------------------------------------------------------------------- diff --git a/python/pyarrow/memory.pxd b/python/pyarrow/memory.pxd index 3079ccb..bb1af85 100644 --- a/python/pyarrow/memory.pxd +++ b/python/pyarrow/memory.pxd @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -from pyarrow.includes.libarrow cimport CMemoryPool +from pyarrow.includes.libarrow cimport CMemoryPool, CLoggingMemoryPool cdef class MemoryPool: @@ -24,4 +24,7 @@ cdef class MemoryPool: cdef init(self, CMemoryPool* pool) +cdef class LoggingMemoryPool(MemoryPool): + pass + cdef CMemoryPool* maybe_unbox_memory_pool(MemoryPool memory_pool) http://git-wip-us.apache.org/repos/asf/arrow/blob/02bdbf48/python/pyarrow/memory.pyx ---------------------------------------------------------------------- diff --git a/python/pyarrow/memory.pyx b/python/pyarrow/memory.pyx index 18a6de4..98dbf66 100644 --- a/python/pyarrow/memory.pyx +++ b/python/pyarrow/memory.pyx @@ -19,7 +19,7 @@ # distutils: language = c++ # cython: embedsignature = True -from pyarrow.includes.libarrow cimport CMemoryPool +from pyarrow.includes.libarrow cimport CMemoryPool, CLoggingMemoryPool from pyarrow.includes.pyarrow cimport set_default_memory_pool, get_memory_pool cdef class MemoryPool: @@ -35,6 +35,9 @@ cdef CMemoryPool* maybe_unbox_memory_pool(MemoryPool memory_pool): else: return memory_pool.pool +cdef class LoggingMemoryPool(MemoryPool): + pass + def default_pool(): cdef: MemoryPool pool = MemoryPool()
