nealrichardson commented on a change in pull request #11032:
URL: https://github.com/apache/arrow/pull/11032#discussion_r724534863
##########
File path: r/R/duckdb.R
##########
@@ -118,3 +118,38 @@ duckdb_disconnector <- function(con, tbl_name) {
})
environment()
}
+
+#' Create an Arrow object from others
+#'
+#' This can be used in pipelines that pass data back and forth between Arrow
and
+#' other processes (like DuckDB).
+#'
+#' @param .data the object to be converted
+#' @param as_table should the results be materialized as a table? (default:
TRUE)
+#'
+#' @return an `arrow_dplyr_query` object, to be used in dplyr pipelines.
+#' @export
+#'
+#' @examplesIf getFromNamespace("run_duckdb_examples", "arrow")()
+#' library(dplyr)
+#'
+#' ds <- InMemoryDataset$create(mtcars)
+#'
+#' ds %>%
+#' filter(mpg < 30) %>%
+#' to_duckdb() %>%
+#' group_by(cyl) %>%
+#' summarize(mean_mpg = mean(mpg, na.rm = TRUE)) %>%
+#' to_arrow() %>%
+#' collect()
+to_arrow <- function(.data, as_table = TRUE) {
Review comment:
Were you going to remove this argument before merging?
##########
File path: r/R/query-engine.R
##########
@@ -59,6 +59,10 @@ ExecPlan <- R6Class("ExecPlan",
Scan = function(dataset) {
# Handle arrow_dplyr_query
if (inherits(dataset, "arrow_dplyr_query")) {
+ if(inherits(dataset$.data, "RecordBatchReader")) {
Review comment:
```suggestion
if (inherits(dataset$.data, "RecordBatchReader")) {
```
##########
File path: r/R/query-engine.R
##########
@@ -59,6 +59,10 @@ ExecPlan <- R6Class("ExecPlan",
Scan = function(dataset) {
# Handle arrow_dplyr_query
if (inherits(dataset, "arrow_dplyr_query")) {
+ if(inherits(dataset$.data, "RecordBatchReader")) {
+ return(ExecNode_ReadFromRecordBatchReader(self, dataset$.data))
+ }
Review comment:
This is a duckdb issue right?
##########
File path: r/R/query-engine.R
##########
@@ -84,6 +88,10 @@ ExecPlan <- R6Class("ExecPlan",
ExecNode_Scan(self, dataset, filter, colnames %||% character(0))
},
Build = function(.data) {
+ if (inherits(.data, "ExecNode")) {
Review comment:
Is this case valid still? I think .data will never be an ExecNode
##########
File path: r/src/compute-exec.cpp
##########
@@ -246,4 +247,22 @@ std::shared_ptr<compute::ExecNode> ExecNode_Join(
std::move(left_out_refs),
std::move(right_out_refs)});
}
+// [[arrow::export]]
+std::shared_ptr<compute::ExecNode> ExecNode_ReadFromRecordBatchReader(
+ const std::shared_ptr<compute::ExecPlan>& plan,
+ const std::shared_ptr<arrow::RecordBatchReader>& reader) {
+ arrow::compute::SourceNodeOptions options{
+ /*output_schema=*/reader->schema(),
+ /*generator=*/ValueOrStop(compute::MakeReaderGenerator(reader,
arrow::internal::GetCpuThreadPool()))
+ };
+
+ return MakeExecNodeOrStop("source", plan.get(), {}, options);
+}
+
#endif
+
+
+
+
+
+
Review comment:
```suggestion
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
