HuaHuaY commented on code in PR #54840:
URL: https://github.com/apache/arrow/pull/54840#discussion_r4476473689
##########
cpp/src/parquet/arrow/reader.cc:
##########
@@ -1307,6 +1336,34 @@ class RowGroupGenerator {
});
}
+ static ::arrow::Future<RecordBatchGenerator> ReadOneRowGroupInBatches(
+ ::arrow::internal::Executor* cpu_executor,
std::shared_ptr<FileReaderImpl> self,
+ const int row_group, const std::vector<int>& column_indices) {
+ if (!cpu_executor) {
+ cpu_executor = ::arrow::internal::GetCpuThreadPool();
+ }
+ ARROW_ASSIGN_OR_RAISE(
+ auto batch_reader,
+ self->GetRecordBatchReaderInternal({row_group}, column_indices,
cpu_executor));
+ // Threaded Next() waits for column tasks on the CPU executor, so
coordinate it on
+ // the I/O executor. Serial Next() performs CPU work directly on the CPU
executor.
+ auto next_executor = self->properties().use_threads()
+ ? self->properties().io_context().executor()
+ : cpu_executor;
+ auto shared_reader =
std::shared_ptr<RecordBatchReader>(std::move(batch_reader));
+ RecordBatchGenerator generator =
+ [self = std::move(self), shared_reader, next_executor,
+ cpu_executor]() ->
::arrow::Future<std::shared_ptr<::arrow::RecordBatch>> {
+ auto next = ::arrow::DeferNotOk(
+ next_executor->Submit([self, shared_reader] { return
shared_reader->Next(); }));
Review Comment:
Can we add some comments to explain why we need to capture an unused
variable `self` here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
