EnricoMi commented on code in PR #44470:
URL: https://github.com/apache/arrow/pull/44470#discussion_r2079069585
##########
cpp/src/arrow/dataset/file_test.cc:
##########
@@ -353,6 +361,153 @@ TEST_F(TestFileSystemDataset, WriteProjected) {
}
}
+// This kernel delays execution for some specific scalar values,
+// which guarantees the writing phase sees out-of-order exec batches
+Status delay(compute::KernelContext* ctx, const compute::ExecSpan& batch,
+ compute::ExecResult* out) {
+ const ArraySpan& input = batch[0].array;
+ const auto* input_values = input.GetValues<uint32_t>(1);
+ uint8_t* output_values = out->array_span()->buffers[1].data;
+
+ // Boolean data is stored in 1 bit per value
+ for (int64_t i = 0; i < input.length; ++i) {
+ if (input_values[i] % 16 == 0) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ }
+ bit_util::SetBitTo(output_values, i, true);
+ }
+
+ return Status::OK();
+}
+
+// A fragment with start=0 will defer ScanBatchesAsync returning a batch
generator
+// This guarantees a dataset of multiple fragments produces out-of-order
batches
+class MockFragment : public Fragment {
+ public:
+ explicit MockFragment(uint32_t start, int64_t rows_per_batch, int
num_batches,
+ const std::shared_ptr<Schema>& schema)
+ : Fragment(compute::literal(true), schema),
+ start_(start),
+ rows_per_batch_(rows_per_batch),
+ num_batches_(num_batches) {}
+
+ Result<RecordBatchGenerator> ScanBatchesAsync(
+ const std::shared_ptr<ScanOptions>& options) override {
+ // One fragment requires some more time than others
Review Comment:
The mock fragment does not know and care if it is the first fragment in the
dataset, the mock dataset knows that. The mock fragment only implements that a
fragment can defer returning the generator.
Let's rephrase this line to
```suggest
// Fragment with start_=0 defers returning the generator
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
