Repository: arrow Updated Branches: refs/heads/master 331be4923 -> 00df40cea
http://git-wip-us.apache.org/repos/asf/arrow/blob/00df40ce/python/src/pyarrow/io.h ---------------------------------------------------------------------- diff --git a/python/src/pyarrow/io.h b/python/src/pyarrow/io.h index 4cb010f..a603e81 100644 --- a/python/src/pyarrow/io.h +++ b/python/src/pyarrow/io.h @@ -20,17 +20,17 @@ #include "arrow/io/interfaces.h" #include "arrow/io/memory.h" +#include "arrow/util/visibility.h" #include "pyarrow/config.h" #include "pyarrow/common.h" -#include "pyarrow/visibility.h" namespace arrow { + class MemoryPool; -} -namespace pyarrow { +namespace py { // A common interface to a Python file-like object. Must acquire GIL before // calling any methods @@ -39,31 +39,31 @@ class PythonFile { PythonFile(PyObject* file); ~PythonFile(); - arrow::Status Close(); - arrow::Status Seek(int64_t position, int whence); - arrow::Status Read(int64_t nbytes, PyObject** out); - arrow::Status Tell(int64_t* position); - arrow::Status Write(const uint8_t* data, int64_t nbytes); + Status Close(); + Status Seek(int64_t position, int whence); + Status Read(int64_t nbytes, PyObject** out); + Status Tell(int64_t* position); + Status Write(const uint8_t* data, int64_t nbytes); private: PyObject* file_; }; -class PYARROW_EXPORT PyReadableFile : public arrow::io::ReadableFileInterface { +class ARROW_EXPORT PyReadableFile : public io::ReadableFileInterface { public: explicit PyReadableFile(PyObject* file); virtual ~PyReadableFile(); - arrow::Status Close() override; + Status Close() override; - arrow::Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override; - arrow::Status Read(int64_t nbytes, std::shared_ptr<arrow::Buffer>* out) override; + Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* out) override; + Status Read(int64_t nbytes, std::shared_ptr<Buffer>* out) override; - arrow::Status GetSize(int64_t* size) override; + Status GetSize(int64_t* size) override; - arrow::Status Seek(int64_t position) override; + Status Seek(int64_t position) override; - arrow::Status Tell(int64_t* position) override; + Status Tell(int64_t* position) override; bool supports_zero_copy() const override; @@ -71,21 +71,21 @@ class PYARROW_EXPORT PyReadableFile : public arrow::io::ReadableFileInterface { std::unique_ptr<PythonFile> file_; }; -class PYARROW_EXPORT PyOutputStream : public arrow::io::OutputStream { +class ARROW_EXPORT PyOutputStream : public io::OutputStream { public: explicit PyOutputStream(PyObject* file); virtual ~PyOutputStream(); - arrow::Status Close() override; - arrow::Status Tell(int64_t* position) override; - arrow::Status Write(const uint8_t* data, int64_t nbytes) override; + Status Close() override; + Status Tell(int64_t* position) override; + Status Write(const uint8_t* data, int64_t nbytes) override; private: std::unique_ptr<PythonFile> file_; }; // A zero-copy reader backed by a PyBytes object -class PYARROW_EXPORT PyBytesReader : public arrow::io::BufferReader { +class ARROW_EXPORT PyBytesReader : public io::BufferReader { public: explicit PyBytesReader(PyObject* obj); virtual ~PyBytesReader(); @@ -93,6 +93,7 @@ class PYARROW_EXPORT PyBytesReader : public arrow::io::BufferReader { // TODO(wesm): seekable output files -} // namespace pyarrow +} // namespace py +} // namespace arrow #endif // PYARROW_IO_H http://git-wip-us.apache.org/repos/asf/arrow/blob/00df40ce/python/src/pyarrow/numpy_interop.h ---------------------------------------------------------------------- diff --git a/python/src/pyarrow/numpy_interop.h b/python/src/pyarrow/numpy_interop.h index 6326527..57f3328 100644 --- a/python/src/pyarrow/numpy_interop.h +++ b/python/src/pyarrow/numpy_interop.h @@ -42,7 +42,8 @@ #include <numpy/arrayobject.h> #include <numpy/ufuncobject.h> -namespace pyarrow { +namespace arrow { +namespace py { inline int import_numpy() { #ifdef NUMPY_IMPORT_ARRAY @@ -53,6 +54,7 @@ inline int import_numpy() { return 0; } -} // namespace pyarrow +} // namespace py +} // namespace arrow #endif // PYARROW_NUMPY_INTEROP_H http://git-wip-us.apache.org/repos/asf/arrow/blob/00df40ce/python/src/pyarrow/type_traits.h ---------------------------------------------------------------------- diff --git a/python/src/pyarrow/type_traits.h b/python/src/pyarrow/type_traits.h new file mode 100644 index 0000000..f4604d7 --- /dev/null +++ b/python/src/pyarrow/type_traits.h @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <Python.h> + +#include <cstdint> + +#include "pyarrow/numpy_interop.h" + +#include "arrow/builder.h" +#include "arrow/type.h" + +namespace arrow { +namespace py { + +template <int TYPE> +struct npy_traits {}; + +template <> +struct npy_traits<NPY_BOOL> { + typedef uint8_t value_type; + using TypeClass = BooleanType; + using BuilderClass = BooleanBuilder; + + static constexpr bool supports_nulls = false; + static inline bool isnull(uint8_t v) { return false; } +}; + +#define NPY_INT_DECL(TYPE, CapType, T) \ + template <> \ + struct npy_traits<NPY_##TYPE> { \ + typedef T value_type; \ + using TypeClass = CapType##Type; \ + using BuilderClass = CapType##Builder; \ + \ + static constexpr bool supports_nulls = false; \ + static inline bool isnull(T v) { return false; } \ + }; + +NPY_INT_DECL(INT8, Int8, int8_t); +NPY_INT_DECL(INT16, Int16, int16_t); +NPY_INT_DECL(INT32, Int32, int32_t); +NPY_INT_DECL(INT64, Int64, int64_t); + +NPY_INT_DECL(UINT8, UInt8, uint8_t); +NPY_INT_DECL(UINT16, UInt16, uint16_t); +NPY_INT_DECL(UINT32, UInt32, uint32_t); +NPY_INT_DECL(UINT64, UInt64, uint64_t); + +#if NPY_INT64 != NPY_LONGLONG +NPY_INT_DECL(LONGLONG, Int64, int64_t); +NPY_INT_DECL(ULONGLONG, UInt64, uint64_t); +#endif + +template <> +struct npy_traits<NPY_FLOAT32> { + typedef float value_type; + using TypeClass = FloatType; + using BuilderClass = FloatBuilder; + + static constexpr bool supports_nulls = true; + + static inline bool isnull(float v) { return v != v; } +}; + +template <> +struct npy_traits<NPY_FLOAT64> { + typedef double value_type; + using TypeClass = DoubleType; + using BuilderClass = DoubleBuilder; + + static constexpr bool supports_nulls = true; + + static inline bool isnull(double v) { return v != v; } +}; + +template <> +struct npy_traits<NPY_DATETIME> { + typedef int64_t value_type; + using TypeClass = TimestampType; + using BuilderClass = TimestampBuilder; + + static constexpr bool supports_nulls = true; + + static inline bool isnull(int64_t v) { + // NaT = -2**63 + // = -0x8000000000000000 + // = -9223372036854775808; + // = std::numeric_limits<int64_t>::min() + return v == std::numeric_limits<int64_t>::min(); + } +}; + +template <> +struct npy_traits<NPY_OBJECT> { + typedef PyObject* value_type; + static constexpr bool supports_nulls = true; +}; + +template <int TYPE> +struct arrow_traits {}; + +template <> +struct arrow_traits<Type::BOOL> { + static constexpr int npy_type = NPY_BOOL; + static constexpr bool supports_nulls = false; + static constexpr bool is_boolean = true; + static constexpr bool is_numeric_not_nullable = false; + static constexpr bool is_numeric_nullable = false; +}; + +#define INT_DECL(TYPE) \ + template <> \ + struct arrow_traits<Type::TYPE> { \ + static constexpr int npy_type = NPY_##TYPE; \ + static constexpr bool supports_nulls = false; \ + static constexpr double na_value = NAN; \ + static constexpr bool is_boolean = false; \ + static constexpr bool is_numeric_not_nullable = true; \ + static constexpr bool is_numeric_nullable = false; \ + typedef typename npy_traits<NPY_##TYPE>::value_type T; \ + }; + +INT_DECL(INT8); +INT_DECL(INT16); +INT_DECL(INT32); +INT_DECL(INT64); +INT_DECL(UINT8); +INT_DECL(UINT16); +INT_DECL(UINT32); +INT_DECL(UINT64); + +template <> +struct arrow_traits<Type::FLOAT> { + static constexpr int npy_type = NPY_FLOAT32; + static constexpr bool supports_nulls = true; + static constexpr float na_value = NAN; + static constexpr bool is_boolean = false; + static constexpr bool is_numeric_not_nullable = false; + static constexpr bool is_numeric_nullable = true; + typedef typename npy_traits<NPY_FLOAT32>::value_type T; +}; + +template <> +struct arrow_traits<Type::DOUBLE> { + static constexpr int npy_type = NPY_FLOAT64; + static constexpr bool supports_nulls = true; + static constexpr double na_value = NAN; + static constexpr bool is_boolean = false; + static constexpr bool is_numeric_not_nullable = false; + static constexpr bool is_numeric_nullable = true; + typedef typename npy_traits<NPY_FLOAT64>::value_type T; +}; + +static constexpr int64_t kPandasTimestampNull = std::numeric_limits<int64_t>::min(); + +template <> +struct arrow_traits<Type::TIMESTAMP> { + static constexpr int npy_type = NPY_DATETIME; + static constexpr bool supports_nulls = true; + static constexpr int64_t na_value = kPandasTimestampNull; + static constexpr bool is_boolean = false; + static constexpr bool is_numeric_not_nullable = false; + static constexpr bool is_numeric_nullable = true; + typedef typename npy_traits<NPY_DATETIME>::value_type T; +}; + +template <> +struct arrow_traits<Type::DATE> { + static constexpr int npy_type = NPY_DATETIME; + static constexpr bool supports_nulls = true; + static constexpr int64_t na_value = kPandasTimestampNull; + static constexpr bool is_boolean = false; + static constexpr bool is_numeric_not_nullable = false; + static constexpr bool is_numeric_nullable = true; + typedef typename npy_traits<NPY_DATETIME>::value_type T; +}; + +template <> +struct arrow_traits<Type::STRING> { + static constexpr int npy_type = NPY_OBJECT; + static constexpr bool supports_nulls = true; + static constexpr bool is_boolean = false; + static constexpr bool is_numeric_not_nullable = false; + static constexpr bool is_numeric_nullable = false; +}; + +template <> +struct arrow_traits<Type::BINARY> { + static constexpr int npy_type = NPY_OBJECT; + static constexpr bool supports_nulls = true; + static constexpr bool is_boolean = false; + static constexpr bool is_numeric_not_nullable = false; + static constexpr bool is_numeric_nullable = false; +}; + +} // namespace py +} // namespace arrow http://git-wip-us.apache.org/repos/asf/arrow/blob/00df40ce/python/src/pyarrow/util/datetime.h ---------------------------------------------------------------------- diff --git a/python/src/pyarrow/util/datetime.h b/python/src/pyarrow/util/datetime.h index 9ffa691..f704a96 100644 --- a/python/src/pyarrow/util/datetime.h +++ b/python/src/pyarrow/util/datetime.h @@ -21,7 +21,8 @@ #include <Python.h> #include <datetime.h> -namespace pyarrow { +namespace arrow { +namespace py { inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) { struct tm date = {0}; @@ -35,6 +36,7 @@ inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) { return lrint(difftime(mktime(&date), mktime(&epoch)) * 1000); } -} // namespace pyarrow +} // namespace py +} // namespace arrow #endif // PYARROW_UTIL_DATETIME_H http://git-wip-us.apache.org/repos/asf/arrow/blob/00df40ce/python/src/pyarrow/util/test_main.cc ---------------------------------------------------------------------- diff --git a/python/src/pyarrow/util/test_main.cc b/python/src/pyarrow/util/test_main.cc index 02e9a54..d8d1d03 100644 --- a/python/src/pyarrow/util/test_main.cc +++ b/python/src/pyarrow/util/test_main.cc @@ -26,7 +26,7 @@ int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); Py_Initialize(); - pyarrow::import_numpy(); + arrow::py::import_numpy(); int ret = RUN_ALL_TESTS(); http://git-wip-us.apache.org/repos/asf/arrow/blob/00df40ce/python/src/pyarrow/visibility.h ---------------------------------------------------------------------- diff --git a/python/src/pyarrow/visibility.h b/python/src/pyarrow/visibility.h deleted file mode 100644 index 9f0c13b..0000000 --- a/python/src/pyarrow/visibility.h +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef PYARROW_VISIBILITY_H -#define PYARROW_VISIBILITY_H - -#if defined(_WIN32) || defined(__CYGWIN__) -#define PYARROW_EXPORT __declspec(dllexport) -#else // Not Windows -#ifndef PYARROW_EXPORT -#define PYARROW_EXPORT __attribute__((visibility("default"))) -#endif -#ifndef PYARROW_NO_EXPORT -#define PYARROW_NO_EXPORT __attribute__((visibility("hidden"))) -#endif -#endif // Non-Windows - -#endif // PYARROW_VISIBILITY_H
