Repository: arrow Updated Branches: refs/heads/master 6aed18f96 -> f442879d3
ARROW-624: [C++] Restore MakePrimitiveArray function, use in feather.cc I verified locally the parquet-cpp test suite passes again Author: Wes McKinney <wes.mckin...@twosigma.com> Closes #378 from wesm/ARROW-624 and squashes the following commits: 023df9b [Wes McKinney] Use passed offset in MakePrimitiveArray 30a553e [Wes McKinney] Restore MakePrimitiveArray function, use in Feather, verify fixes parquet test suite Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/f442879d Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/f442879d Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/f442879d Branch: refs/heads/master Commit: f442879d3c791d86fb0fdfa098a72329843f5baf Parents: 6aed18f Author: Wes McKinney <wes.mckin...@twosigma.com> Authored: Tue Mar 14 09:17:30 2017 +0100 Committer: Uwe L. Korn <uw...@xhochy.com> Committed: Tue Mar 14 09:17:30 2017 +0100 ---------------------------------------------------------------------- cpp/src/arrow/api.h | 3 +++ cpp/src/arrow/ipc/feather.cc | 8 +------- cpp/src/arrow/loader.cc | 28 +++++++++++++++++++++++----- cpp/src/arrow/loader.h | 10 ++++++++++ 4 files changed, 37 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/f442879d/cpp/src/arrow/api.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/api.h b/cpp/src/arrow/api.h index 51437d8..3bc8666 100644 --- a/cpp/src/arrow/api.h +++ b/cpp/src/arrow/api.h @@ -24,7 +24,10 @@ #include "arrow/buffer.h" #include "arrow/builder.h" #include "arrow/column.h" +#include "arrow/compare.h" +#include "arrow/loader.h" #include "arrow/memory_pool.h" +#include "arrow/pretty_print.h" #include "arrow/schema.h" #include "arrow/status.h" #include "arrow/table.h" http://git-wip-us.apache.org/repos/asf/arrow/blob/f442879d/cpp/src/arrow/ipc/feather.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/ipc/feather.cc b/cpp/src/arrow/ipc/feather.cc index 13dfa58..1d165ac 100644 --- a/cpp/src/arrow/ipc/feather.cc +++ b/cpp/src/arrow/ipc/feather.cc @@ -331,7 +331,6 @@ class TableReader::TableReaderImpl { std::shared_ptr<DataType> type; RETURN_NOT_OK(GetDataType(meta, metadata_type, metadata, &type)); - std::vector<FieldMetadata> fields(1); std::vector<std::shared_ptr<Buffer>> buffers; // Buffer data from the source (may or may not perform a copy depending on @@ -357,12 +356,7 @@ class TableReader::TableReaderImpl { } buffers.push_back(SliceBuffer(buffer, offset, buffer->size() - offset)); - - fields[0].length = meta->length(); - fields[0].null_count = meta->null_count(); - fields[0].offset = 0; - - return LoadArray(type, fields, buffers, out); + return MakePrimitiveArray(type, buffers, meta->length(), meta->null_count(), 0, out); } bool HasDescription() const { return metadata_->HasDescription(); } http://git-wip-us.apache.org/repos/asf/arrow/blob/f442879d/cpp/src/arrow/loader.cc ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/loader.cc b/cpp/src/arrow/loader.cc index 3cb51ae..0b3ee1c 100644 --- a/cpp/src/arrow/loader.cc +++ b/cpp/src/arrow/loader.cc @@ -235,8 +235,8 @@ class ArrayLoader : public TypeVisitor { std::shared_ptr<Array> result_; }; -Status ARROW_EXPORT LoadArray(const std::shared_ptr<DataType>& type, - ArrayComponentSource* source, std::shared_ptr<Array>* out) { +Status LoadArray(const std::shared_ptr<DataType>& type, ArrayComponentSource* source, + std::shared_ptr<Array>* out) { ArrayLoaderContext context; context.source = source; context.field_index = context.buffer_index = 0; @@ -244,8 +244,8 @@ Status ARROW_EXPORT LoadArray(const std::shared_ptr<DataType>& type, return LoadArray(type, &context, out); } -Status ARROW_EXPORT LoadArray(const std::shared_ptr<DataType>& type, - ArrayLoaderContext* context, std::shared_ptr<Array>* out) { +Status LoadArray(const std::shared_ptr<DataType>& type, ArrayLoaderContext* context, + std::shared_ptr<Array>* out) { ArrayLoader loader(type, context); RETURN_NOT_OK(loader.Load(out)); @@ -275,11 +275,29 @@ class InMemorySource : public ArrayComponentSource { const std::vector<std::shared_ptr<Buffer>>& buffers_; }; -Status ARROW_EXPORT LoadArray(const std::shared_ptr<DataType>& type, +Status LoadArray(const std::shared_ptr<DataType>& type, const std::vector<FieldMetadata>& fields, const std::vector<std::shared_ptr<Buffer>>& buffers, std::shared_ptr<Array>* out) { InMemorySource source(fields, buffers); return LoadArray(type, &source, out); } +Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, int64_t length, + const std::shared_ptr<Buffer>& data, const std::shared_ptr<Buffer>& null_bitmap, + int64_t null_count, int64_t offset, std::shared_ptr<Array>* out) { + std::vector<std::shared_ptr<Buffer>> buffers = {null_bitmap, data}; + return MakePrimitiveArray(type, buffers, length, null_count, offset, out); +} + +Status MakePrimitiveArray(const std::shared_ptr<DataType>& type, + const std::vector<std::shared_ptr<Buffer>>& buffers, int64_t length, + int64_t null_count, int64_t offset, std::shared_ptr<Array>* out) { + std::vector<FieldMetadata> fields(1); + fields[0].length = length; + fields[0].null_count = null_count; + fields[0].offset = offset; + + return LoadArray(type, fields, buffers, out); +} + } // namespace arrow http://git-wip-us.apache.org/repos/asf/arrow/blob/f442879d/cpp/src/arrow/loader.h ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/loader.h b/cpp/src/arrow/loader.h index b4949f2..f116d64 100644 --- a/cpp/src/arrow/loader.h +++ b/cpp/src/arrow/loader.h @@ -84,6 +84,16 @@ Status ARROW_EXPORT LoadArray(const std::shared_ptr<DataType>& type, const std::vector<FieldMetadata>& fields, const std::vector<std::shared_ptr<Buffer>>& buffers, std::shared_ptr<Array>* out); +/// Create new arrays for logical types that are backed by primitive arrays. +Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type, + int64_t length, const std::shared_ptr<Buffer>& data, + const std::shared_ptr<Buffer>& null_bitmap, int64_t null_count, int64_t offset, + std::shared_ptr<Array>* out); + +Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type, + const std::vector<std::shared_ptr<Buffer>>& buffers, int64_t length, + int64_t null_count, int64_t offset, std::shared_ptr<Array>* out); + } // namespace arrow #endif // ARROW_LOADER_H