This is an automated email from the ASF dual-hosted git repository.
ashb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 91423baf366 Fix breeze down crashing when a start-airflow container is
running (#73955)
91423baf366 is described below
commit 91423baf3660084c818b5935d0e73e5e94ff3b16
Author: Andrew Chang <[email protected]>
AuthorDate: Wed Sep 30 22:16:54 2026 +0800
Fix breeze down crashing when a start-airflow container is running (#73955)
* Fix breeze down crashing when a start-airflow container is running
breeze start-airflow starts its container with docker compose run --rm,
so the container deletes itself as soon as it stops. breeze down stopped
it and then failed to remove it with "No such container", leaving the
project network and volumes behind. The container can also take longer
than the stop timeout to exit, which made the stop itself fail.
* Tell the user that breeze down is stopping containers
A breeze start-airflow container can take tens of seconds to exit, and
the output of the stop and wait commands is hidden, so breeze down looked
stuck during that time.
---
.../airflow_breeze/utils/docker_command_utils.py | 26 +++++--
dev/breeze/tests/test_docker_command_utils.py | 86 ++++++++++++++++++++--
2 files changed, 99 insertions(+), 13 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index d5d8c7814b0..28a0f08ce2c 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -1064,18 +1064,28 @@ def bring_compose_projects_down(
projects.add(project)
if targets["container"]:
- result = run_command(
- ["docker", "container", "stop", *targets["container"]],
- check=not stale_only,
- capture_output=stale_only,
- )
- failed |= result.returncode != 0
+ # A `breeze shell` or `breeze start-airflow` container can exit later
than `docker container stop`
+ # waits for, and it removes itself once it exits. So the exit is
awaited separately, and only the
+ # containers that are still there afterwards are removed.
+ console_print("[info]Stopping Breeze containers, this can take a
while.[/]")
+ for action in ("stop", "wait"):
+ run_command(
+ ["docker", "container", action, *targets["container"]],
check=False, capture_output=True
+ )
+ remaining, discovery_failed = _get_compose_resources("container",
stale_only=stale_only)
+ failed |= discovery_failed
+ remaining_ids = {identifier for identifier, _ in remaining}
+ targets["container"] = [
+ identifier for identifier in targets["container"] if identifier in
remaining_ids
+ ]
for kind, identifiers in targets.items():
if not identifiers:
continue
cmd = ["docker", kind, "rm"]
- if kind == "container" and not preserve_volumes:
- cmd.append("--volumes")
+ if kind == "container":
+ cmd.append("--force")
+ if not preserve_volumes:
+ cmd.append("--volumes")
result = run_command(
[*cmd, *identifiers],
check=not stale_only,
diff --git a/dev/breeze/tests/test_docker_command_utils.py
b/dev/breeze/tests/test_docker_command_utils.py
index 2e147d0127c..28b8dc60221 100644
--- a/dev/breeze/tests/test_docker_command_utils.py
+++ b/dev/breeze/tests/test_docker_command_utils.py
@@ -424,7 +424,7 @@ def docker_resources():
elif action == "inspect":
output = json.dumps(resources[kind])
else:
- assert action in ("stop", "rm")
+ assert action in ("stop", "wait", "rm")
output = ""
return subprocess.CompletedProcess(cmd, 0, stdout=output, stderr="")
@@ -502,7 +502,7 @@ def test_down_selects_checkout_stale_or_explicit_projects(
stops = [cmd for cmd in commands if cmd[1:3] == ["container", "stop"]]
assert {arg for cmd in stops for arg in cmd[3:]} ==
{f"{project}-container" for project in expected}
first_removal = next(i for i, cmd in enumerate(commands) if cmd[2] ==
"rm")
- assert all(cmd[2] in ("ls", "inspect", "stop") for cmd in
commands[:first_removal])
+ assert all(cmd[2] in ("ls", "inspect", "stop", "wait") for cmd in
commands[:first_removal])
removal = next(cmd for cmd in commands if cmd[1:3] == ["container",
"rm"])
assert ("--volumes" in removal) is not preserve_volumes
@@ -612,14 +612,19 @@ def
test_startup_cleanup_continues_after_docker_failures(docker_resources, tmp_p
return result
run.side_effect = fail
- assert docker_command_utils.bring_compose_projects_down(stale_only=True)
== []
- assert capsys.readouterr().out.count("Unable to clean up some
deleted-worktree resources") == 1
+ stop_failed = failed_action == "stop"
+ assert docker_command_utils.bring_compose_projects_down(stale_only=True)
== (
+ ["stale"] if stop_failed else []
+ )
+ assert capsys.readouterr().out.count("Unable to clean up some
deleted-worktree resources") == (
+ 0 if stop_failed else 1
+ )
commands = [c.args[0] for c in run.call_args_list]
if failed_action == "ls":
assert all(cmd[2] == "ls" for cmd in commands)
else:
assert ["docker", "volume", "rm", "database"] in commands
- assert ["docker", "container", "rm", "--volumes", "remaining"] in
commands
+ assert ["docker", "container", "rm", "--force", "--volumes",
"remaining"] in commands
assert not any("vanished" in cmd for cmd in commands if cmd[2] in
("stop", "rm"))
@@ -672,6 +677,77 @@ def
test_down_stops_after_container_removal_failure(docker_resources):
assert not any(c.args[0][1:3] == ["volume", "rm"] for c in
run.call_args_list)
+def test_down_force_removes_containers_that_outlive_stop(docker_resources,
capsys):
+ resources, run = docker_resources
+ labels = {"com.docker.compose.project": "breeze"}
+ resources["container"] = [{"Id": "container", "Config": {"Labels":
labels}}]
+ docker = run.side_effect
+
+ def fail(cmd, **kwargs):
+ if cmd[1:3] == ["container", "stop"]:
+ if kwargs.get("check"):
+ raise subprocess.CalledProcessError(1, cmd)
+ return subprocess.CompletedProcess(cmd, 1, stdout="", stderr="did
not receive an exit event")
+ return docker(cmd, **kwargs)
+
+ run.side_effect = fail
+ assert docker_command_utils.bring_compose_projects_down() == ["breeze"]
+
+ assert "Stopping Breeze containers" in capsys.readouterr().out
+ assert [c.args[0] for c in run.call_args_list if c.args[0][2] in ("stop",
"wait", "rm")] == [
+ ["docker", "container", "stop", "container"],
+ ["docker", "container", "wait", "container"],
+ ["docker", "container", "rm", "--force", "--volumes", "container"],
+ ]
+
+
+def
test_down_skips_removal_of_containers_that_removed_themselves(docker_resources):
+ resources, run = docker_resources
+ labels = {"com.docker.compose.project": "breeze"}
+ resources["container"] = [{"Id": "container", "Config": {"Labels":
labels}}]
+ resources["volume"] = [{"Name": "database", "Labels": labels}]
+ docker = run.side_effect
+
+ def remove_on_stop(cmd, **kwargs):
+ if cmd[1:3] == ["container", "stop"]:
+ resources["container"].clear()
+ return docker(cmd, **kwargs)
+
+ run.side_effect = remove_on_stop
+ assert docker_command_utils.bring_compose_projects_down() == ["breeze"]
+
+ assert [c.args[0] for c in run.call_args_list if c.args[0][2] == "rm"] == [
+ ["docker", "volume", "rm", "database"]
+ ]
+
+
+def test_startup_cleanup_reports_containers_that_cannot_be_listed_after_stop(
+ docker_resources, tmp_path, capsys
+):
+ resources, run = docker_resources
+ labels = {
+ "com.docker.compose.project": "stale",
+ "org.apache.airflow.breeze": "true",
+ "org.apache.airflow.breeze.worktree": str(tmp_path / "deleted"),
+ }
+ resources["container"] = [{"Id": "remaining", "Config": {"Labels":
labels}}]
+ docker = run.side_effect
+ stopped = False
+
+ def fail_listing_after_stop(cmd, **kwargs):
+ nonlocal stopped
+ result = docker(cmd, **kwargs)
+ if cmd[1:3] == ["container", "stop"]:
+ stopped = True
+ if stopped and cmd[1:3] == ["container", "ls"]:
+ result.returncode = 1
+ return result
+
+ run.side_effect = fail_listing_after_stop
+ assert docker_command_utils.bring_compose_projects_down(stale_only=True)
== []
+ assert capsys.readouterr().out.count("Unable to clean up some
deleted-worktree resources") == 1
+
+
def
test_down_removes_volumes_even_when_a_shared_network_is_still_in_use(docker_resources):
resources, run = docker_resources
labels = {"com.docker.compose.project": "breeze"}
