This is an automated email from the ASF dual-hosted git repository.
westonpace pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new b46ef0f2c4 ARROW-18367: [C++] Enable the creation of named table
relations (#14681)
b46ef0f2c4 is described below
commit b46ef0f2c4109553789003e12783bd08ae1ec39d
Author: Jianshen Liu <[email protected]>
AuthorDate: Thu Dec 8 12:21:29 2022 -0800
ARROW-18367: [C++] Enable the creation of named table relations (#14681)
- changes also include fixes from running IWYU over the directory
cpp/src/arrow/engine/substrait
Lead-authored-by: Jianshen Liu <[email protected]>
Co-authored-by: Weston Pace <[email protected]>
Signed-off-by: Weston Pace <[email protected]>
---
cpp/src/arrow/compute/exec/options.h | 15 +++
cpp/src/arrow/compute/exec/source_node.cc | 9 ++
.../arrow/engine/substrait/expression_internal.cc | 27 +++++
.../arrow/engine/substrait/expression_internal.h | 10 +-
cpp/src/arrow/engine/substrait/ext_test.cc | 22 ++--
cpp/src/arrow/engine/substrait/extension_set.cc | 7 +-
cpp/src/arrow/engine/substrait/extension_set.h | 3 +-
cpp/src/arrow/engine/substrait/extension_types.cc | 6 +-
cpp/src/arrow/engine/substrait/extension_types.h | 3 +-
cpp/src/arrow/engine/substrait/function_test.cc | 19 +++-
cpp/src/arrow/engine/substrait/plan_internal.cc | 16 ++-
cpp/src/arrow/engine/substrait/plan_internal.h | 7 +-
.../arrow/engine/substrait/protobuf_test_util.cc | 1 +
cpp/src/arrow/engine/substrait/relation.h | 1 +
.../arrow/engine/substrait/relation_internal.cc | 66 ++++++++++--
cpp/src/arrow/engine/substrait/relation_internal.h | 10 +-
cpp/src/arrow/engine/substrait/serde.cc | 19 ++--
cpp/src/arrow/engine/substrait/serde_test.cc | 118 ++++++++++++++++++---
.../arrow/engine/substrait/test_plan_builder.cc | 13 ++-
cpp/src/arrow/engine/substrait/test_plan_builder.h | 9 +-
cpp/src/arrow/engine/substrait/type_internal.cc | 16 ++-
cpp/src/arrow/engine/substrait/type_internal.h | 6 +-
cpp/src/arrow/engine/substrait/util.cc | 16 ++-
cpp/src/arrow/engine/substrait/util.h | 13 ++-
cpp/src/arrow/flight/sql/acero_test.cc | 1 +
25 files changed, 354 insertions(+), 79 deletions(-)
diff --git a/cpp/src/arrow/compute/exec/options.h
b/cpp/src/arrow/compute/exec/options.h
index 8600b11348..2f0f009c01 100644
--- a/cpp/src/arrow/compute/exec/options.h
+++ b/cpp/src/arrow/compute/exec/options.h
@@ -85,6 +85,21 @@ class ARROW_EXPORT TableSourceNodeOptions : public
ExecNodeOptions {
int64_t max_batch_size;
};
+/// \brief Define a lazy resolved Arrow table.
+///
+/// The table uniquely identified by the names can typically be resolved at
the time when
+/// the plan is to be consumed.
+///
+/// This node is for serialization purposes only and can never be executed.
+class ARROW_EXPORT NamedTableNodeOptions : public ExecNodeOptions {
+ public:
+ NamedTableNodeOptions(std::vector<std::string> names,
std::shared_ptr<Schema> schema)
+ : names(std::move(names)), schema(schema) {}
+
+ std::vector<std::string> names;
+ std::shared_ptr<Schema> schema;
+};
+
/// \brief An extended Source node which accepts a schema
///
/// ItMaker is a maker of an iterator of tabular data.
diff --git a/cpp/src/arrow/compute/exec/source_node.cc
b/cpp/src/arrow/compute/exec/source_node.cc
index 3fd4c6fd5b..3f8f4c9834 100644
--- a/cpp/src/arrow/compute/exec/source_node.cc
+++ b/cpp/src/arrow/compute/exec/source_node.cc
@@ -426,6 +426,14 @@ struct ArrayVectorSourceNode
const char ArrayVectorSourceNode::kKindName[] = "ArrayVectorSourceNode";
+Result<compute::ExecNode*> MakeNamedTableNode(compute::ExecPlan* plan,
+ std::vector<compute::ExecNode*>
inputs,
+ const compute::ExecNodeOptions&
options) {
+ return Status::Invalid(
+ "The named table node is for serialization purposes only and can never
be "
+ "converted into an exec plan or executed");
+}
+
} // namespace
namespace internal {
@@ -436,6 +444,7 @@ void RegisterSourceNode(ExecFactoryRegistry* registry) {
DCHECK_OK(registry->AddFactory("record_batch_source",
RecordBatchSourceNode::Make));
DCHECK_OK(registry->AddFactory("exec_batch_source",
ExecBatchSourceNode::Make));
DCHECK_OK(registry->AddFactory("array_vector_source",
ArrayVectorSourceNode::Make));
+ DCHECK_OK(registry->AddFactory("named_table", MakeNamedTableNode));
}
} // namespace internal
diff --git a/cpp/src/arrow/engine/substrait/expression_internal.cc
b/cpp/src/arrow/engine/substrait/expression_internal.cc
index b8b545febc..dab8786885 100644
--- a/cpp/src/arrow/engine/substrait/expression_internal.cc
+++ b/cpp/src/arrow/engine/substrait/expression_internal.cc
@@ -19,16 +19,43 @@
#include "arrow/engine/substrait/expression_internal.h"
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstring>
+#include <functional>
#include <memory>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <type_traits>
#include <utility>
+#include <vector>
+#include <google/protobuf/descriptor.h>
+
+#include "arrow/array/array_base.h"
+#include "arrow/array/array_nested.h"
+#include "arrow/array/array_primitive.h"
+#include "arrow/array/util.h"
+#include "arrow/buffer.h"
#include "arrow/builder.h"
+#include "arrow/compute/api_scalar.h"
#include "arrow/compute/exec/expression.h"
#include "arrow/compute/exec/expression_internal.h"
+#include "arrow/engine/substrait/extension_set.h"
#include "arrow/engine/substrait/extension_types.h"
+#include "arrow/engine/substrait/options.h"
#include "arrow/engine/substrait/type_internal.h"
#include "arrow/result.h"
+#include "arrow/scalar.h"
#include "arrow/status.h"
+#include "arrow/type.h"
+#include "arrow/util/checked_cast.h"
+#include "arrow/util/decimal.h"
+#include "arrow/util/endian.h"
+#include "arrow/util/logging.h"
+#include "arrow/util/small_vector.h"
#include "arrow/util/string.h"
#include "arrow/visit_scalar_inline.h"
diff --git a/cpp/src/arrow/engine/substrait/expression_internal.h
b/cpp/src/arrow/engine/substrait/expression_internal.h
index f132afc0c1..e947537dd1 100644
--- a/cpp/src/arrow/engine/substrait/expression_internal.h
+++ b/cpp/src/arrow/engine/substrait/expression_internal.h
@@ -19,19 +19,21 @@
#pragma once
-#include <utility>
+#include <memory>
#include "arrow/compute/type_fwd.h"
-#include "arrow/engine/substrait/extension_set.h"
-#include "arrow/engine/substrait/options.h"
+#include "arrow/datum.h"
+#include "arrow/engine/substrait/type_fwd.h"
#include "arrow/engine/substrait/visibility.h"
-#include "arrow/type_fwd.h"
+#include "arrow/result.h"
#include "substrait/algebra.pb.h" // IWYU pragma: export
namespace arrow {
namespace engine {
+class SubstraitCall;
+
ARROW_ENGINE_EXPORT
Result<compute::Expression> FromProto(const substrait::Expression&, const
ExtensionSet&,
const ConversionOptions&);
diff --git a/cpp/src/arrow/engine/substrait/ext_test.cc
b/cpp/src/arrow/engine/substrait/ext_test.cc
index 525af7c946..e7aa7a8e8d 100644
--- a/cpp/src/arrow/engine/substrait/ext_test.cc
+++ b/cpp/src/arrow/engine/substrait/ext_test.cc
@@ -15,16 +15,24 @@
// specific language governing permissions and limitations
// under the License.
-#include "arrow/engine/substrait/extension_set.h"
-#include "arrow/engine/substrait/util.h"
-
-#include <google/protobuf/descriptor.h>
-#include <google/protobuf/util/json_util.h>
-#include <google/protobuf/util/type_resolver_util.h>
+#include <cstddef>
+#include <memory>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <tuple>
+#include <vector>
+
+#include <gmock/gmock.h>
#include <gtest/gtest.h>
+#include "arrow/engine/substrait/extension_set.h"
+#include "arrow/engine/substrait/util.h"
+#include "arrow/result.h"
+#include "arrow/status.h"
#include "arrow/testing/gtest_util.h"
-#include "arrow/testing/matchers.h"
+#include "arrow/type_fwd.h"
+#include "arrow/util/checked_cast.h"
using testing::ElementsAre;
using testing::Eq;
diff --git a/cpp/src/arrow/engine/substrait/extension_set.cc
b/cpp/src/arrow/engine/substrait/extension_set.cc
index 1c0a927156..6cd022dc4a 100644
--- a/cpp/src/arrow/engine/substrait/extension_set.cc
+++ b/cpp/src/arrow/engine/substrait/extension_set.cc
@@ -17,14 +17,19 @@
#include "arrow/engine/substrait/extension_set.h"
+#include <algorithm>
+#include <iterator>
#include <list>
#include <memory>
#include <sstream>
#include <unordered_set>
-#include "arrow/engine/substrait/expression_internal.h"
+#include "arrow/engine/substrait/options.h"
+#include "arrow/type.h"
+#include "arrow/type_fwd.h"
#include "arrow/util/hash_util.h"
#include "arrow/util/hashing.h"
+#include "arrow/util/logging.h"
#include "arrow/util/string.h"
namespace arrow {
diff --git a/cpp/src/arrow/engine/substrait/extension_set.h
b/cpp/src/arrow/engine/substrait/extension_set.h
index 12aa40115b..8b6b5d5159 100644
--- a/cpp/src/arrow/engine/substrait/extension_set.h
+++ b/cpp/src/arrow/engine/substrait/extension_set.h
@@ -32,9 +32,10 @@
#include "arrow/compute/api_aggregate.h"
#include "arrow/compute/exec/expression.h"
-#include "arrow/engine/substrait/options.h"
+#include "arrow/engine/substrait/type_fwd.h"
#include "arrow/engine/substrait/visibility.h"
#include "arrow/result.h"
+#include "arrow/status.h"
#include "arrow/type_fwd.h"
#include "arrow/util/macros.h"
diff --git a/cpp/src/arrow/engine/substrait/extension_types.cc
b/cpp/src/arrow/engine/substrait/extension_types.cc
index 6a89e3cf98..fcc722e9d9 100644
--- a/cpp/src/arrow/engine/substrait/extension_types.cc
+++ b/cpp/src/arrow/engine/substrait/extension_types.cc
@@ -17,10 +17,14 @@
#include "arrow/engine/substrait/extension_types.h"
+#include <cstdint>
+#include <string>
#include <string_view>
#include "arrow/engine/simple_extension_type_internal.h"
-#include "arrow/util/hashing.h"
+#include "arrow/result.h"
+#include "arrow/type_fwd.h"
+#include "arrow/util/reflection_internal.h"
namespace arrow {
diff --git a/cpp/src/arrow/engine/substrait/extension_types.h
b/cpp/src/arrow/engine/substrait/extension_types.h
index 3b08084c75..28a4898a87 100644
--- a/cpp/src/arrow/engine/substrait/extension_types.h
+++ b/cpp/src/arrow/engine/substrait/extension_types.h
@@ -19,9 +19,10 @@
#pragma once
+#include <cstdint>
+#include <memory>
#include <optional>
-#include "arrow/compute/function.h"
#include "arrow/engine/substrait/visibility.h"
#include "arrow/type_fwd.h"
diff --git a/cpp/src/arrow/engine/substrait/function_test.cc
b/cpp/src/arrow/engine/substrait/function_test.cc
index 2c00e70ff7..4dc7ce8327 100644
--- a/cpp/src/arrow/engine/substrait/function_test.cc
+++ b/cpp/src/arrow/engine/substrait/function_test.cc
@@ -15,28 +15,37 @@
// specific language governing permissions and limitations
// under the License.
+#include <algorithm>
+#include <cstddef>
#include <memory>
#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <utility>
#include <vector>
-#include <gmock/gmock-matchers.h>
+#include <gmock/gmock.h>
+#include <gtest/gtest-matchers.h>
#include <gtest/gtest.h>
-#include "arrow/array.h"
#include "arrow/array/builder_binary.h"
+#include "arrow/compute/api_vector.h"
#include "arrow/compute/cast.h"
+#include "arrow/compute/exec/exec_plan.h"
#include "arrow/compute/exec/options.h"
#include "arrow/compute/exec/util.h"
+#include "arrow/datum.h"
#include "arrow/engine/substrait/extension_set.h"
-#include "arrow/engine/substrait/plan_internal.h"
+#include "arrow/engine/substrait/options.h"
#include "arrow/engine/substrait/serde.h"
#include "arrow/engine/substrait/test_plan_builder.h"
-#include "arrow/engine/substrait/type_internal.h"
#include "arrow/record_batch.h"
+#include "arrow/result.h"
+#include "arrow/status.h"
#include "arrow/table.h"
#include "arrow/testing/future_util.h"
#include "arrow/testing/gtest_util.h"
-#include "arrow/type.h"
+#include "arrow/type_fwd.h"
namespace arrow {
diff --git a/cpp/src/arrow/engine/substrait/plan_internal.cc
b/cpp/src/arrow/engine/substrait/plan_internal.cc
index e675a2c2ab..fff0f7563c 100644
--- a/cpp/src/arrow/engine/substrait/plan_internal.cc
+++ b/cpp/src/arrow/engine/substrait/plan_internal.cc
@@ -17,16 +17,24 @@
#include "arrow/engine/substrait/plan_internal.h"
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <utility>
+
+#include "arrow/compute/type_fwd.h"
#include "arrow/config.h"
-#include "arrow/dataset/plan.h"
#include "arrow/engine/substrait/relation_internal.h"
+#include "arrow/engine/substrait/type_fwd.h"
#include "arrow/result.h"
+#include "arrow/util/checked_cast.h"
#include "arrow/util/hashing.h"
-#include "arrow/util/logging.h"
+#include "arrow/util/macros.h"
#include "arrow/util/unreachable.h"
-#include <memory>
-#include <unordered_map>
+#include "substrait/extensions/extensions.pb.h"
namespace arrow {
diff --git a/cpp/src/arrow/engine/substrait/plan_internal.h
b/cpp/src/arrow/engine/substrait/plan_internal.h
index 7d6dd37528..235bf1a6ce 100644
--- a/cpp/src/arrow/engine/substrait/plan_internal.h
+++ b/cpp/src/arrow/engine/substrait/plan_internal.h
@@ -19,11 +19,14 @@
#pragma once
-#include "arrow/compute/exec/exec_plan.h"
+#include <memory>
+
+#include "arrow/compute/type_fwd.h"
#include "arrow/engine/substrait/extension_set.h"
#include "arrow/engine/substrait/options.h"
#include "arrow/engine/substrait/visibility.h"
-#include "arrow/type_fwd.h"
+#include "arrow/result.h"
+#include "arrow/status.h"
#include "substrait/plan.pb.h" // IWYU pragma: export
diff --git a/cpp/src/arrow/engine/substrait/protobuf_test_util.cc
b/cpp/src/arrow/engine/substrait/protobuf_test_util.cc
index d99622912c..20b1bb3270 100644
--- a/cpp/src/arrow/engine/substrait/protobuf_test_util.cc
+++ b/cpp/src/arrow/engine/substrait/protobuf_test_util.cc
@@ -18,6 +18,7 @@
#include <gtest/gtest.h>
#include <google/protobuf/message_lite.h>
+#include <google/protobuf/stubs/common.h>
namespace arrow {
namespace engine {
diff --git a/cpp/src/arrow/engine/substrait/relation.h
b/cpp/src/arrow/engine/substrait/relation.h
index c2103a0b9e..06bb3d5854 100644
--- a/cpp/src/arrow/engine/substrait/relation.h
+++ b/cpp/src/arrow/engine/substrait/relation.h
@@ -20,6 +20,7 @@
#include <memory>
#include "arrow/compute/exec/exec_plan.h"
+#include "arrow/engine/substrait/visibility.h"
#include "arrow/type_fwd.h"
namespace arrow {
diff --git a/cpp/src/arrow/engine/substrait/relation_internal.cc
b/cpp/src/arrow/engine/substrait/relation_internal.cc
index 019e402a44..83c36ee9bb 100644
--- a/cpp/src/arrow/engine/substrait/relation_internal.cc
+++ b/cpp/src/arrow/engine/substrait/relation_internal.cc
@@ -17,24 +17,46 @@
#include "arrow/engine/substrait/relation_internal.h"
-#include "arrow/compute/api_scalar.h"
+#include <cstdint>
+#include <functional>
+#include <iterator>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <type_traits>
+#include <utility>
+#include <variant>
+#include <vector>
+
+#include "arrow/compute/api_aggregate.h"
+#include "arrow/compute/exec/exec_plan.h"
+#include "arrow/compute/exec/expression.h"
#include "arrow/compute/exec/options.h"
+#include "arrow/compute/kernel.h"
+#include "arrow/dataset/dataset.h"
+#include "arrow/dataset/discovery.h"
#include "arrow/dataset/file_base.h"
#include "arrow/dataset/file_ipc.h"
#include "arrow/dataset/file_parquet.h"
#include "arrow/dataset/plan.h"
#include "arrow/dataset/scanner.h"
+#include "arrow/datum.h"
#include "arrow/engine/substrait/expression_internal.h"
+#include "arrow/engine/substrait/extension_set.h"
+#include "arrow/engine/substrait/options.h"
+#include "arrow/engine/substrait/relation.h"
#include "arrow/engine/substrait/type_internal.h"
+#include "arrow/filesystem/filesystem.h"
#include "arrow/filesystem/localfs.h"
-#include "arrow/filesystem/path_util.h"
+#include "arrow/filesystem/type_fwd.h"
#include "arrow/filesystem/util_internal.h"
+#include "arrow/io/type_fwd.h"
+#include "arrow/status.h"
+#include "arrow/type.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/string.h"
#include "arrow/util/uri.h"
-#include <memory>
-
namespace arrow {
using internal::checked_cast;
@@ -638,6 +660,10 @@ Result<std::shared_ptr<Schema>> ExtractSchemaToBind(const
compute::Declaration&
} else if (declr.factory_name == "filter") {
auto input_declr = std::get<compute::Declaration>(declr.inputs[0]);
ARROW_ASSIGN_OR_RAISE(bind_schema, ExtractSchemaToBind(input_declr));
+ } else if (declr.factory_name == "named_table") {
+ const auto& opts =
+ checked_cast<const compute::NamedTableNodeOptions&>(*declr.options);
+ bind_schema = opts.schema;
} else if (declr.factory_name == "sink") {
// Note that the sink has no output_schema
return bind_schema;
@@ -648,6 +674,30 @@ Result<std::shared_ptr<Schema>> ExtractSchemaToBind(const
compute::Declaration&
return bind_schema;
}
+Result<std::unique_ptr<substrait::ReadRel>> NamedTableRelationConverter(
+ const std::shared_ptr<Schema>& schema, const compute::Declaration&
declaration,
+ ExtensionSet* ext_set, const ConversionOptions& conversion_options) {
+ auto read_rel = std::make_unique<substrait::ReadRel>();
+ const auto& named_table_options =
+ checked_cast<const
compute::NamedTableNodeOptions&>(*declaration.options);
+
+ // set schema
+ ARROW_ASSIGN_OR_RAISE(auto named_struct, ToProto(*schema, ext_set,
conversion_options));
+ read_rel->set_allocated_base_schema(named_struct.release());
+
+ if (named_table_options.names.empty()) {
+ return Status::Invalid("Table names cannot be empty");
+ }
+
+ auto read_rel_tn = std::make_unique<substrait::ReadRel::NamedTable>();
+ for (auto& name : named_table_options.names) {
+ read_rel_tn->add_names(name);
+ }
+ read_rel->set_allocated_named_table(read_rel_tn.release());
+
+ return std::move(read_rel);
+}
+
Result<std::unique_ptr<substrait::ReadRel>> ScanRelationConverter(
const std::shared_ptr<Schema>& schema, const compute::Declaration&
declaration,
ExtensionSet* ext_set, const ConversionOptions& conversion_options) {
@@ -662,8 +712,7 @@ Result<std::unique_ptr<substrait::ReadRel>>
ScanRelationConverter(
}
// set schema
- ARROW_ASSIGN_OR_RAISE(auto named_struct,
- ToProto(*dataset->schema(), ext_set,
conversion_options));
+ ARROW_ASSIGN_OR_RAISE(auto named_struct, ToProto(*schema, ext_set,
conversion_options));
read_rel->set_allocated_base_schema(named_struct.release());
// set local files
@@ -743,6 +792,11 @@ Status SerializeAndCombineRelations(const
compute::Declaration& declaration,
auto filter_rel,
FilterRelationConverter(schema, declaration, ext_set,
conversion_options));
(*rel)->set_allocated_filter(filter_rel.release());
+ } else if (factory_name == "named_table") {
+ ARROW_ASSIGN_OR_RAISE(
+ auto read_rel,
+ NamedTableRelationConverter(schema, declaration, ext_set,
conversion_options));
+ (*rel)->set_allocated_read(read_rel.release());
} else if (factory_name == "sink") {
// Generally when a plan is deserialized the declaration will be a sink
declaration.
// Since there is no Sink relation in substrait, this function would be
recursively
diff --git a/cpp/src/arrow/engine/substrait/relation_internal.h
b/cpp/src/arrow/engine/substrait/relation_internal.h
index df7804b679..17153f5365 100644
--- a/cpp/src/arrow/engine/substrait/relation_internal.h
+++ b/cpp/src/arrow/engine/substrait/relation_internal.h
@@ -19,13 +19,13 @@
#pragma once
-#include "arrow/compute/exec/exec_plan.h"
-#include "arrow/engine/substrait/extension_types.h"
-#include "arrow/engine/substrait/options.h"
+#include <memory>
+
+#include "arrow/compute/type_fwd.h"
#include "arrow/engine/substrait/relation.h"
-#include "arrow/engine/substrait/serde.h"
+#include "arrow/engine/substrait/type_fwd.h"
#include "arrow/engine/substrait/visibility.h"
-#include "arrow/type_fwd.h"
+#include "arrow/result.h"
#include "substrait/algebra.pb.h" // IWYU pragma: export
diff --git a/cpp/src/arrow/engine/substrait/serde.cc
b/cpp/src/arrow/engine/substrait/serde.cc
index b90bc98aab..1a82e23dc9 100644
--- a/cpp/src/arrow/engine/substrait/serde.cc
+++ b/cpp/src/arrow/engine/substrait/serde.cc
@@ -17,8 +17,19 @@
#include "arrow/engine/substrait/serde.h"
+#include <cstdint>
+#include <type_traits>
#include <utility>
+#include <google/protobuf/descriptor.h>
+#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
+#include <google/protobuf/message.h>
+#include <google/protobuf/stubs/status.h>
+#include <google/protobuf/util/json_util.h>
+#include <google/protobuf/util/message_differencer.h>
+#include <google/protobuf/util/type_resolver.h>
+#include <google/protobuf/util/type_resolver_util.h>
+
#include "arrow/buffer.h"
#include "arrow/compute/exec/exec_plan.h"
#include "arrow/compute/exec/expression.h"
@@ -27,18 +38,12 @@
#include "arrow/engine/substrait/expression_internal.h"
#include "arrow/engine/substrait/extension_set.h"
#include "arrow/engine/substrait/plan_internal.h"
+#include "arrow/engine/substrait/relation.h"
#include "arrow/engine/substrait/relation_internal.h"
#include "arrow/engine/substrait/type_fwd.h"
#include "arrow/engine/substrait/type_internal.h"
#include "arrow/type.h"
-#include <google/protobuf/descriptor.h>
-#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
-#include <google/protobuf/message.h>
-#include <google/protobuf/util/json_util.h>
-#include <google/protobuf/util/message_differencer.h>
-#include <google/protobuf/util/type_resolver_util.h>
-
namespace arrow {
namespace engine {
diff --git a/cpp/src/arrow/engine/substrait/serde_test.cc
b/cpp/src/arrow/engine/substrait/serde_test.cc
index 210edcf75e..fb91702871 100644
--- a/cpp/src/arrow/engine/substrait/serde_test.cc
+++ b/cpp/src/arrow/engine/substrait/serde_test.cc
@@ -15,36 +15,64 @@
// specific language governing permissions and limitations
// under the License.
-#include <google/protobuf/descriptor.h>
-#include <google/protobuf/util/json_util.h>
-#include <google/protobuf/util/type_resolver_util.h>
+#include <algorithm>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <optional>
+#include <type_traits>
+#include <utility>
+#include <variant>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest-matchers.h>
#include <gtest/gtest.h>
+#include "arrow/buffer.h"
+#include "arrow/compute/api_scalar.h"
+#include "arrow/compute/api_vector.h"
+#include "arrow/compute/exec.h"
#include "arrow/compute/exec/exec_plan.h"
+#include "arrow/compute/exec/expression.h"
#include "arrow/compute/exec/expression_internal.h"
+#include "arrow/compute/exec/options.h"
+#include "arrow/compute/registry.h"
+#include "arrow/compute/type_fwd.h"
+#include "arrow/dataset/dataset.h"
+#include "arrow/dataset/discovery.h"
#include "arrow/dataset/file_base.h"
#include "arrow/dataset/file_ipc.h"
-#include "arrow/dataset/file_parquet.h"
+#include "arrow/dataset/partition.h"
#include "arrow/dataset/plan.h"
#include "arrow/dataset/scanner.h"
+#include "arrow/datum.h"
+#include "arrow/engine/substrait/extension_set.h"
#include "arrow/engine/substrait/extension_types.h"
+#include "arrow/engine/substrait/options.h"
#include "arrow/engine/substrait/serde.h"
#include "arrow/engine/substrait/util.h"
-
+#include "arrow/filesystem/filesystem.h"
#include "arrow/filesystem/localfs.h"
#include "arrow/filesystem/mockfs.h"
#include "arrow/filesystem/test_util.h"
-#include "arrow/io/compressed.h"
-#include "arrow/io/memory.h"
+#include "arrow/io/type_fwd.h"
+#include "arrow/ipc/options.h"
#include "arrow/ipc/writer.h"
+#include "arrow/scalar.h"
+#include "arrow/table.h"
+#include "arrow/testing/future_util.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/testing/matchers.h"
-#include "arrow/util/key_value_metadata.h"
-
-#include "parquet/arrow/writer.h"
-
+#include "arrow/type.h"
+#include "arrow/type_fwd.h"
+#include "arrow/util/async_generator_fwd.h"
+#include "arrow/util/checked_cast.h"
+#include "arrow/util/decimal.h"
+#include "arrow/util/future.h"
#include "arrow/util/hash_util.h"
-#include "arrow/util/hashing.h"
+#include "arrow/util/io_util.h"
+#include "arrow/util/iterator.h"
+#include "arrow/util/key_value_metadata.h"
using testing::ElementsAre;
using testing::Eq;
@@ -366,14 +394,20 @@ TEST(Substrait, NamedStruct) {
/*ignore_unknown_fields=*/false));
EXPECT_THAT(DeserializeSchema(*buf, ext_set), Raises(StatusCode::Invalid));
- // no schema metadata allowed
- EXPECT_THAT(SerializeSchema(Schema({}, key_value_metadata({{"ext",
"yes"}})), &ext_set),
+ ConversionOptions conversion_options;
+ conversion_options.strictness = ConversionStrictness::EXACT_ROUNDTRIP;
+
+ // no schema metadata allowed with EXACT_ROUNDTRIP
+ EXPECT_THAT(SerializeSchema(Schema({}, key_value_metadata({{"ext",
"yes"}})), &ext_set,
+ conversion_options),
Raises(StatusCode::Invalid));
- // no schema metadata allowed
+ ASSERT_OK(SerializeSchema(Schema({}, key_value_metadata({{"ext", "yes"}})),
&ext_set));
+
+ // no field metadata allowed with EXACT_ROUNDTRIP
EXPECT_THAT(
SerializeSchema(Schema({field("a", int32(), key_value_metadata({{"ext",
"yes"}}))}),
- &ext_set),
+ &ext_set, conversion_options),
Raises(StatusCode::Invalid));
}
@@ -2037,7 +2071,6 @@ TEST(Substrait, AggregateBadPhase) {
}
TEST(SubstraitRoundTrip, BasicPlan) {
- compute::ExecContext exec_context;
arrow::dataset::internal::Initialize();
auto dummy_schema = schema(
@@ -2260,6 +2293,57 @@ TEST(SubstraitRoundTrip, BasicPlanEndToEnd) {
EXPECT_TRUE(expected_table->Equals(*rnd_trp_table));
}
+TEST(SubstraitRoundTrip, FilterNamedTable) {
+ compute::ExecContext exec_context;
+ arrow::dataset::internal::Initialize();
+
+ const std::vector<std::string> table_names{"table", "1"};
+ const auto dummy_schema =
+ schema({field("A", int32()), field("B", int32()), field("C", int32())});
+ auto filter = compute::equal(compute::field_ref("A"),
compute::field_ref("B"));
+
+ auto declarations = compute::Declaration::Sequence(
+ {compute::Declaration({"named_table",
+ compute::NamedTableNodeOptions{table_names,
dummy_schema},
+ "n"}),
+ compute::Declaration({"filter", compute::FilterNodeOptions{filter},
"f"})});
+
+ ExtensionSet ext_set{};
+ ASSERT_OK_AND_ASSIGN(auto serialized_plan, SerializePlan(declarations,
&ext_set));
+
+ // creating a dummy dataset using a dummy table
+ auto input_table = TableFromJSON(dummy_schema, {R"([
+ [1, 1, 10],
+ [3, 5, 20],
+ [4, 1, 30],
+ [2, 1, 40],
+ [5, 5, 50],
+ [2, 2, 60]
+ ])"});
+
+ NamedTableProvider table_provider =
+ [&input_table, &table_names](
+ const std::vector<std::string>& names) ->
Result<compute::Declaration> {
+ if (table_names != names) {
+ return Status::Invalid("Table name mismatch");
+ }
+ std::shared_ptr<compute::ExecNodeOptions> options =
+ std::make_shared<compute::TableSourceNodeOptions>(input_table);
+ return compute::Declaration("table_source", {}, std::move(options),
"mock_source");
+ };
+ ConversionOptions conversion_options;
+ conversion_options.named_table_provider = std::move(table_provider);
+
+ auto expected_table = TableFromJSON(dummy_schema, {R"([
+ [1, 1, 10],
+ [5, 5, 50],
+ [2, 2, 60]
+ ])"});
+
+ CheckRoundTripResult(std::move(dummy_schema), std::move(expected_table),
exec_context,
+ serialized_plan, {}, conversion_options);
+}
+
TEST(SubstraitRoundTrip, ProjectRel) {
compute::ExecContext exec_context;
auto dummy_schema =
diff --git a/cpp/src/arrow/engine/substrait/test_plan_builder.cc
b/cpp/src/arrow/engine/substrait/test_plan_builder.cc
index 2643b5c469..62f4361a61 100644
--- a/cpp/src/arrow/engine/substrait/test_plan_builder.cc
+++ b/cpp/src/arrow/engine/substrait/test_plan_builder.cc
@@ -17,17 +17,22 @@
#include "arrow/engine/substrait/test_plan_builder.h"
+#include <cstddef>
#include <cstdint>
+#include <limits>
#include <memory>
+#include <utility>
-#include "arrow/compute/exec/exec_plan.h"
+#include "arrow/buffer.h"
+#include "arrow/engine/substrait/extension_set.h"
+#include "arrow/engine/substrait/options.h"
#include "arrow/engine/substrait/plan_internal.h"
#include "arrow/engine/substrait/type_internal.h"
-#include "arrow/util/macros.h"
+#include "arrow/status.h"
+#include "arrow/table.h"
+#include "arrow/type_fwd.h"
#include "substrait/algebra.pb.h"
-#include "substrait/plan.pb.h"
-#include "substrait/type.pb.h"
namespace arrow {
namespace engine {
diff --git a/cpp/src/arrow/engine/substrait/test_plan_builder.h
b/cpp/src/arrow/engine/substrait/test_plan_builder.h
index 5f6629e905..8af156ea61 100644
--- a/cpp/src/arrow/engine/substrait/test_plan_builder.h
+++ b/cpp/src/arrow/engine/substrait/test_plan_builder.h
@@ -30,14 +30,15 @@
#include <unordered_map>
#include <vector>
-#include "arrow/buffer.h"
-#include "arrow/engine/substrait/extension_set.h"
+#include "arrow/engine/substrait/visibility.h"
#include "arrow/result.h"
-#include "arrow/table.h"
-#include "arrow/type.h"
+#include "arrow/type_fwd.h"
namespace arrow {
namespace engine {
+
+struct Id;
+
namespace internal {
/// \brief Create a scan->project->sink plan for tests
diff --git a/cpp/src/arrow/engine/substrait/type_internal.cc
b/cpp/src/arrow/engine/substrait/type_internal.cc
index 16032df67d..f56aa19a04 100644
--- a/cpp/src/arrow/engine/substrait/type_internal.cc
+++ b/cpp/src/arrow/engine/substrait/type_internal.cc
@@ -17,15 +17,23 @@
#include "arrow/engine/substrait/type_internal.h"
+#include <array>
+#include <cstdint>
#include <memory>
+#include <optional>
#include <string>
+#include <tuple>
+#include <type_traits>
#include <vector>
+#include "arrow/engine/substrait/extension_set.h"
#include "arrow/engine/substrait/extension_types.h"
+#include "arrow/engine/substrait/options.h"
+#include "arrow/extension_type.h"
#include "arrow/result.h"
#include "arrow/status.h"
#include "arrow/type.h"
-#include "arrow/util/logging.h"
+#include "arrow/type_fwd.h"
#include "arrow/visit_type_inline.h"
namespace arrow {
@@ -471,7 +479,8 @@ void ToProtoGetDepthFirstNames(const FieldVector& fields,
Result<std::unique_ptr<::substrait::NamedStruct>> ToProto(
const Schema& schema, ExtensionSet* ext_set,
const ConversionOptions& conversion_options) {
- if (schema.metadata()) {
+ if (conversion_options.strictness == ConversionStrictness::EXACT_ROUNDTRIP &&
+ schema.metadata() != nullptr) {
return Status::Invalid("::substrait::NamedStruct does not support schema
metadata");
}
@@ -486,7 +495,8 @@ Result<std::unique_ptr<::substrait::NamedStruct>> ToProto(
types->Reserve(schema.num_fields());
for (const auto& field : schema.fields()) {
- if (field->metadata() != nullptr) {
+ if (conversion_options.strictness == ConversionStrictness::EXACT_ROUNDTRIP
&&
+ field->metadata() != nullptr) {
return Status::Invalid("::substrait::NamedStruct does not support field
metadata");
}
diff --git a/cpp/src/arrow/engine/substrait/type_internal.h
b/cpp/src/arrow/engine/substrait/type_internal.h
index 6db9aea01a..0d53028f49 100644
--- a/cpp/src/arrow/engine/substrait/type_internal.h
+++ b/cpp/src/arrow/engine/substrait/type_internal.h
@@ -19,11 +19,13 @@
#pragma once
+#include <memory>
+#include <string>
#include <utility>
-#include "arrow/engine/substrait/extension_set.h"
-#include "arrow/engine/substrait/options.h"
+#include "arrow/engine/substrait/type_fwd.h"
#include "arrow/engine/substrait/visibility.h"
+#include "arrow/result.h"
#include "arrow/type_fwd.h"
#include "substrait/type.pb.h" // IWYU pragma: export
diff --git a/cpp/src/arrow/engine/substrait/util.cc
b/cpp/src/arrow/engine/substrait/util.cc
index 8db7ff7cb1..87a7a8e364 100644
--- a/cpp/src/arrow/engine/substrait/util.cc
+++ b/cpp/src/arrow/engine/substrait/util.cc
@@ -17,10 +17,24 @@
#include "arrow/engine/substrait/util.h"
+#include <algorithm>
+#include <optional>
+#include <string_view>
+#include <utility>
+
+#include "arrow/buffer.h"
+#include "arrow/compute/exec.h"
#include "arrow/compute/exec/exec_plan.h"
#include "arrow/compute/exec/options.h"
+#include "arrow/compute/type_fwd.h"
+#include "arrow/engine/substrait/extension_set.h"
+#include "arrow/engine/substrait/serde.h"
+#include "arrow/engine/substrait/type_fwd.h"
+#include "arrow/status.h"
+#include "arrow/type_fwd.h"
#include "arrow/util/async_generator.h"
-#include "arrow/util/async_util.h"
+#include "arrow/util/future.h"
+#include "arrow/util/thread_pool.h"
namespace arrow {
diff --git a/cpp/src/arrow/engine/substrait/util.h
b/cpp/src/arrow/engine/substrait/util.h
index a616968d96..5c98c6ff3f 100644
--- a/cpp/src/arrow/engine/substrait/util.h
+++ b/cpp/src/arrow/engine/substrait/util.h
@@ -17,13 +17,18 @@
#pragma once
+#include <functional>
#include <memory>
-#include <optional>
+#include <string>
+#include <vector>
-#include "arrow/compute/registry.h"
-#include "arrow/engine/substrait/api.h"
+#include "arrow/compute/type_fwd.h"
#include "arrow/engine/substrait/options.h"
-#include "arrow/util/iterator.h"
+#include "arrow/engine/substrait/type_fwd.h"
+#include "arrow/engine/substrait/visibility.h"
+#include "arrow/result.h"
+#include "arrow/type_fwd.h"
+#include "arrow/util/macros.h"
namespace arrow {
diff --git a/cpp/src/arrow/flight/sql/acero_test.cc
b/cpp/src/arrow/flight/sql/acero_test.cc
index 9c3f99426f..a8298f57b9 100644
--- a/cpp/src/arrow/flight/sql/acero_test.cc
+++ b/cpp/src/arrow/flight/sql/acero_test.cc
@@ -30,6 +30,7 @@
#include "arrow/flight/sql/example/acero_server.h"
#include "arrow/flight/sql/types.h"
#include "arrow/flight/types.h"
+#include "arrow/scalar.h"
#include "arrow/stl_iterator.h"
#include "arrow/table.h"
#include "arrow/testing/gtest_util.h"
