westonpace commented on code in PR #13518:
URL: https://github.com/apache/arrow/pull/13518#discussion_r923460831
##########
cpp/src/arrow/dataset/scanner_test.cc:
##########
@@ -1377,37 +1381,40 @@ DatasetAndBatches DatasetAndBatchesFromJSON(
fragment_batches.end());
fragments.push_back(std::make_shared<InMemoryFragment>(
physical_schema, std::move(fragment_batches),
- guarantees.empty() ? literal(true) : guarantees[i]));
+ guarantees.empty() ? literal(true) : guarantees[frag_ndx]));
}
+ // then construct ExecBatches that can reference fields from constructed
Fragments
std::vector<compute::ExecBatch> batches;
auto batch_it = record_batches.begin();
- for (size_t fragment_index = 0; fragment_index < fragment_batch_strs.size();
- ++fragment_index) {
- for (size_t batch_index = 0; batch_index <
fragment_batch_strs[fragment_index].size();
- ++batch_index) {
+ for (size_t frag_ndx = 0; frag_ndx < fragment_batch_strs.size(); ++frag_ndx)
{
+ size_t frag_batch_count = fragment_batch_strs[frag_ndx].size();
+
+ for (size_t batch_index = 0; batch_index < frag_batch_count;
++batch_index) {
const auto& batch = *batch_it++;
// the scanned ExecBatches will begin with physical columns
batches.emplace_back(*batch);
- // allow customizing the ExecBatch (e.g. to fill in placeholders for
partition
- // fields)
- if (make_exec_batch) {
- make_exec_batch(&batches.back(), *batch);
+ // augment scanned ExecBatch with columns for this fragment's guarantee
+ if (not guarantees.empty()) {
Review Comment:
```suggestion
if (!guarantees.empty()) {
```
##########
cpp/src/arrow/dataset/scanner_test.cc:
##########
@@ -1377,37 +1381,40 @@ DatasetAndBatches DatasetAndBatchesFromJSON(
fragment_batches.end());
fragments.push_back(std::make_shared<InMemoryFragment>(
physical_schema, std::move(fragment_batches),
- guarantees.empty() ? literal(true) : guarantees[i]));
+ guarantees.empty() ? literal(true) : guarantees[frag_ndx]));
}
+ // then construct ExecBatches that can reference fields from constructed
Fragments
std::vector<compute::ExecBatch> batches;
auto batch_it = record_batches.begin();
- for (size_t fragment_index = 0; fragment_index < fragment_batch_strs.size();
- ++fragment_index) {
- for (size_t batch_index = 0; batch_index <
fragment_batch_strs[fragment_index].size();
- ++batch_index) {
+ for (size_t frag_ndx = 0; frag_ndx < fragment_batch_strs.size(); ++frag_ndx)
{
+ size_t frag_batch_count = fragment_batch_strs[frag_ndx].size();
+
+ for (size_t batch_index = 0; batch_index < frag_batch_count;
++batch_index) {
const auto& batch = *batch_it++;
// the scanned ExecBatches will begin with physical columns
batches.emplace_back(*batch);
- // allow customizing the ExecBatch (e.g. to fill in placeholders for
partition
- // fields)
- if (make_exec_batch) {
- make_exec_batch(&batches.back(), *batch);
+ // augment scanned ExecBatch with columns for this fragment's guarantee
+ if (not guarantees.empty()) {
+ auto extract_result = ExtractKnownFieldValues(guarantees[frag_ndx]);
+ ARROW_WARN_NOT_OK(extract_result.status(), "ExtractKnownFieldValues
failed");
Review Comment:
```suggestion
EXPECT_OK_AND_ASSIGN(auto extract_result,
ExtractKnownFieldValues(guarantees[frag_ndx]));
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
