comphead commented on code in PR #7693:
URL: https://github.com/apache/datafusion-comet/pull/7693#discussion_r4637036476
##########
native/local/src/lib.rs:
##########
@@ -324,7 +348,17 @@ impl PartitionOrder {
.running_limit
.as_ref()
.map_or(self.max_running, |limit| limit().clamp(1,
self.max_running));
- while self.running.len() < limit && self.next < partitions {
+ let mut running = self
+ .running
+ .iter()
+ .filter(|partition| !partition.finished.load(Ordering::Acquire))
+ .count();
+ while running < limit
+ && self.next < partitions
+ && (self.running.len() < limit
+ || self.read_ahead.held.load(Ordering::Acquire) <
self.read_ahead.limit)
Review Comment:
A partition that sends an error now also sets `finished` (the task returns
from the error arm and reaches the drop at the end), so it frees its slot and
`start_partitions` keeps starting the partitions behind it, up to the
read-ahead bound. None of those can be delivered: when the failed partition's
turn comes, `fail` clears `running`, and a consumer that stops earlier never
reaches them either. Before this change the failed partition kept its slot, so
at most `max_running - 1` others ran behind it. With the default 1 GiB
`spark.driver.maxResultSize`, I expect a fast failure behind a slow first
partition to now run most of the remaining partitions for nothing. Could the
task record the failure on `ReadAhead` (for example a `failed` flag set before
`finished`), and `start_partitions` return early once it is set? Every
partition before the failed one is already started, so stopping there cannot
stall delivery.
##########
native/local/tests/execution.rs:
##########
@@ -474,74 +474,174 @@ async fn
partition_order_cancel_releases_running_partitions() {
wait_released(&refs).await;
}
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-async fn partition_order_runs_at_most_the_given_number_of_partitions() {
- let (plan, observations) = observe(ordered_input(6, 3));
- let executed = || observations.executions.lock().unwrap().clone();
- let mut stream = LocalQuery::new(plan, context(), 2, usize::MAX)
- .execute()
- .unwrap();
- // The first two partitions start with the query; the rest wait for their
turn.
- wait_started(&observations, 2).await;
- for _ in 0..3 {
- timeout(DEADLINE, stream.next())
- .await
- .unwrap()
- .unwrap()
- .unwrap();
+/// Partitions planned by a start function, each of which delivers one batch
once its gate opens.
+struct Gated {
+ started: Mutex<Vec<usize>>,
+ gates: Vec<Arc<tokio::sync::Semaphore>>,
+ /// Held by each partition's stream until it is dropped.
+ streams: Vec<Arc<()>>,
+}
+
+impl Gated {
+ fn new(partitions: usize) -> Arc<Self> {
+ Arc::new(Self {
+ started: Mutex::new(Vec::new()),
+ gates: (0..partitions)
+ .map(|_| Arc::new(tokio::sync::Semaphore::new(0)))
+ .collect(),
+ streams: (0..partitions).map(|_| Arc::new(())).collect(),
+ })
}
- assert_eq!(executed(), vec![0, 1]);
- // Finishing the first partition starts the third.
- timeout(DEADLINE, stream.next())
+
+ fn query(self: &Arc<Self>, max_running: usize, read_ahead: usize) ->
LocalQuery {
+ let gated = Arc::clone(self);
+ let start: PartitionStart = Arc::new(move |partition, _| {
+ gated.started.lock().unwrap().push(partition);
+ let gate = Arc::clone(&gated.gates[partition]);
+ let held = Arc::clone(&gated.streams[partition]);
+ let base = partition as i64 * 4;
+ let stream = futures::stream::once(async move {
+ gate.acquire().await.unwrap().forget();
+ Ok(batch(base..base + 4))
+ })
+ .inspect(move |_| {
+ let _ = &held;
+ });
+ Ok(Box::pin(RecordBatchStreamAdapter::new(schema(), stream)))
+ });
+ LocalQuery::from_partitions(
+ schema(),
+ self.gates.len(),
+ start,
+ context(),
+ max_running,
+ read_ahead,
+ )
+ }
+
+ fn open(&self, partition: usize) {
+ self.gates[partition].add_permits(1);
+ }
+
+ fn started(&self) -> Vec<usize> {
+ let mut started = self.started.lock().unwrap().clone();
+ started.sort_unstable();
+ started
+ }
+
+ async fn wait_started(&self, partitions: usize) {
+ timeout(DEADLINE, async {
+ while self.started.lock().unwrap().len() < partitions {
+ tokio::task::yield_now().await;
+ }
+ })
.await
- .unwrap()
- .unwrap()
- .unwrap();
- wait_started(&observations, 3).await;
- let rest: Vec<RecordBatch> = timeout(DEADLINE, stream.try_collect())
+ .expect("partitions did not start");
+ // Long enough for a partition started beyond the bound to show up.
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ assert_eq!(self.started(), (0..partitions).collect::<Vec<_>>());
+ }
+
+ /// Opens every gate and checks that `reader` read every partition in
order.
+ async fn read_in_order(&self, reader:
tokio::task::JoinHandle<Result<Vec<RecordBatch>>>) {
+ for partition in 0..self.gates.len() {
+ self.open(partition);
+ }
+ let batches = timeout(DEADLINE,
reader).await.unwrap().unwrap().unwrap();
+ let expected: Vec<i64> = (0..self.gates.len() as i64 * 4).collect();
+ assert_eq!(ids(&batches), expected);
+ assert_eq!(self.started(), (0..self.gates.len()).collect::<Vec<_>>());
+ }
+
+ async fn wait_dropped(&self, partition: usize) {
+ timeout(DEADLINE, async {
+ while Arc::strong_count(&self.streams[partition]) > 1 {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .expect("a finished partition kept its stream");
+ }
+}
+
+fn ids(batches: &[RecordBatch]) -> Vec<i64> {
+ batches
+ .iter()
+ .flat_map(|batch| {
+ batch
+ .column(1)
+ .as_any()
+ .downcast_ref::<Int64Array>()
+ .unwrap()
+ .values()
+ .to_vec()
+ })
+ .collect()
+}
+
+async fn collect_in_order(
+ gated: &Gated,
+ mut stream: impl futures::Stream<Item = Result<RecordBatch>> + Unpin,
+ from: usize,
+) {
+ for partition in from..gated.gates.len() {
+ gated.open(partition);
+ }
+ let rest: Vec<RecordBatch> = timeout(DEADLINE,
stream.by_ref().try_collect())
.await
.unwrap()
.unwrap();
- assert_eq!(rest.len(), 6 * 3 - 4);
- assert_executed_once(&observations, 6);
+ assert_eq!(
+ ids(&rest),
+ (from as i64 * 4..gated.gates.len() as i64 * 4).collect::<Vec<_>>()
+ );
+ assert_eq!(gated.started(), (0..gated.gates.len()).collect::<Vec<_>>());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-async fn partitions_from_a_start_function_are_planned_as_they_start() {
- let planned = Arc::new(Mutex::new(Vec::new()));
- let (plans, source) = (Arc::clone(&planned), ordered_input(6, 1));
- let start: PartitionStart = Arc::new(move |partition, context| {
- plans.lock().unwrap().push(partition);
- source.execute(partition, context)
- });
- let mut stream = LocalQuery::from_partitions(schema(), 6, start,
context(), 2, usize::MAX)
- .execute()
- .unwrap();
- let planned_now = || planned.lock().unwrap().clone();
- timeout(DEADLINE, async {
- while planned_now().len() < 2 {
- tokio::task::yield_now().await;
- }
- })
- .await
- .expect("the first two partitions were not planned");
- timeout(DEADLINE, stream.next())
+async fn partition_order_runs_at_most_the_given_number_of_partitions() {
+ let gated = Gated::new(6);
+ let mut stream = gated.query(2, usize::MAX).execute().unwrap();
+ // Partitions are planned only as they start, and two run at once.
+ gated.wait_started(2).await;
+ // Finishing the first partition starts the third.
+ gated.open(0);
+ let first = timeout(DEADLINE, stream.next())
.await
.unwrap()
.unwrap()
.unwrap();
- // The third partition is planned only once the first has been delivered.
- let mut first = planned_now();
- first.sort_unstable();
- assert_eq!(first, vec![0, 1]);
- let rest: Vec<RecordBatch> = timeout(DEADLINE, stream.try_collect())
- .await
- .unwrap()
- .unwrap();
- assert_eq!(rest.len(), 5);
- let mut all = planned_now();
- all.sort_unstable();
- assert_eq!(all, (0..6).collect::<Vec<_>>());
+ assert_eq!(ids(&[first]), (0..4).collect::<Vec<_>>());
+ gated.wait_started(3).await;
+ collect_in_order(&gated, stream, 1).await;
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn a_partition_that_finishes_before_its_turn_gives_up_its_slot() {
+ let gated = Gated::new(6);
+ let reader = tokio::spawn(gated.query(2,
usize::MAX).execute().unwrap().try_collect());
+ gated.wait_started(2).await;
+ // While the first partition runs, each later one that finishes drops its
operators and
+ // lets the next one start, and its batch waits for its turn.
+ for partition in 1..4 {
+ gated.open(partition);
+ gated.wait_dropped(partition).await;
+ gated.wait_started(partition + 2).await;
+ }
+ gated.read_in_order(reader).await;
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn finished_partitions_wait_for_their_turn_within_the_read_ahead_limit()
{
+ let gated = Gated::new(6);
+ let reader = tokio::spawn(gated.query(2,
0).execute().unwrap().try_collect());
+ gated.wait_started(2).await;
+ // The second partition finishes, but it holds a batch beyond the
read-ahead limit, so no
+ // partition starts beyond the two that are not delivered.
+ gated.open(1);
+ gated.wait_dropped(1).await;
+ gated.wait_started(2).await;
Review Comment:
With `read_ahead` 0, `held < limit` is never true, so the third partition
cannot start whether or not partition 1's batch is counted in `held`. The test
pins the `running.len() < limit` gate, but not what the comment on line 639
describes. With a limit of 1 byte, the test would fail if the finished
partition's batch were left out of `held`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]