This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 8843de3ed17 Speed up breeze start by caching Python bytecode in a
docker volume (#72567)
8843de3ed17 is described below
commit 8843de3ed17f932b4c47808bdfec1d527e7febdb
Author: Andrew Chang <[email protected]>
AuthorDate: Thu Sep 17 21:33:09 2026 +0800
Speed up breeze start by caching Python bytecode in a docker volume (#72567)
* Speed up breeze shell and start-airflow by caching Python bytecode in a
docker volume
Breeze sets PYTHONDONTWRITEBYTECODE=true for its containers, so every
airflow process recompiles all mounted sources on import, reading them
through the bind mount. On macOS this costs about a second per airflow
CLI call and several seconds per component start.
Writing .pyc files next to the sources would pollute the host checkout
with root-owned files, which is why they were disabled. Pointing
PYTHONPYCACHEPREFIX at an external docker volume keeps the cache out of
the checkout and lets it survive between container runs. Only shell and
start-airflow mount the volume, other containers keep the old behaviour.
* Update dev/breeze/doc/03_developer_tasks.rst
Co-authored-by: Jarek Potiuk <[email protected]>
---------
Co-authored-by: Jarek Potiuk <[email protected]>
---
dev/breeze/doc/03_developer_tasks.rst | 14 ++++++++++++
dev/breeze/doc/images/output_down.svg | 18 +++++++++------
dev/breeze/doc/images/output_down.txt | 2 +-
.../airflow_breeze/commands/developer_commands.py | 11 +++++++++
.../commands/developer_commands_config.py | 1 +
dev/breeze/src/airflow_breeze/global_constants.py | 4 +++-
.../src/airflow_breeze/params/shell_params.py | 15 ++++++++++++-
.../airflow_breeze/utils/docker_command_utils.py | 3 +++
dev/breeze/src/airflow_breeze/utils/path_utils.py | 8 +++++++
dev/breeze/tests/test_docker_command_utils.py | 1 +
dev/breeze/tests/test_shell_params.py | 23 +++++++++++++++++++
scripts/ci/docker-compose/pycache.yml | 26 ++++++++++++++++++++++
12 files changed, 116 insertions(+), 10 deletions(-)
diff --git a/dev/breeze/doc/03_developer_tasks.rst
b/dev/breeze/doc/03_developer_tasks.rst
index 8a9b4459848..f2d510cf590 100644
--- a/dev/breeze/doc/03_developer_tasks.rst
+++ b/dev/breeze/doc/03_developer_tasks.rst
@@ -399,6 +399,20 @@ in ``--from-ref`` and ``--to-ref`` flags.
If the cache gets broken, run ``breeze down --cleanup-mypy-cache`` which
wipes the docker
volume and every per-hook ``.build/mypy-venvs/`` and
``.build/mypy-caches/`` directory.
+.. note::
+
+ Python bytecode (``.pyc``) compiled from the mounted sources inside the
container is written
+ to the ``airflow-pycache-volume`` docker volume (``PYTHONPYCACHEPREFIX``)
rather than next to
+ the sources, so it never shows up in your checkout but survives between
``breeze shell`` and
+ ``breeze start-airflow`` runs. This noticeably speeds up every ``airflow``
command and component
+ start-up, especially on macOS where reading sources through the bind mount
is slow.
+ The cache is safe to share across Python versions: ``.pyc`` file names
keep the
+ interpreter tag (``foo.cpython-310.pyc`` vs ``foo.cpython-312.pyc``), so
bytecode is
+ never reused across versions. It is shared across worktrees, though --
sources always
+ mount at ``/opt/airflow`` and freshness is checked by source mtime and
size, so
+ alternating between worktrees keeps invalidating the other one's entries.
+ Run ``breeze down --cleanup-pycache`` to wipe the volume.
+
.. note::
You cannot change Python version for static checks that are run within
Breeze containers.
diff --git a/dev/breeze/doc/images/output_down.svg
b/dev/breeze/doc/images/output_down.svg
index 84dd611651f..7ddbe85eb32 100644
--- a/dev/breeze/doc/images/output_down.svg
+++ b/dev/breeze/doc/images/output_down.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 635.5999999999999"
xmlns="http://www.w3.org/2000/svg";>
+<svg class="rich-terminal" viewBox="0 0 1482 660.0"
xmlns="http://www.w3.org/2000/svg";>
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -43,7 +43,7 @@
<defs>
<clipPath id="breeze-down-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="584.5999999999999" />
+ <rect x="0" y="0" width="1463.0" height="609.0" />
</clipPath>
<clipPath id="breeze-down-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -114,9 +114,12 @@
<clipPath id="breeze-down-line-22">
<rect x="0" y="538.3" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-down-line-23">
+ <rect x="0" y="562.7" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="633.6" rx="8"/><text class="breeze-down-title"
fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: down</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="658" rx="8"/><text class="breeze-down-title"
fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: down</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -146,10 +149,11 @@
</text><text class="breeze-down-r5" x="0" y="434.8" textLength="12.2"
clip-path="url(#breeze-down-line-17)">│</text><text class="breeze-down-r1"
x="219.6" y="434.8" textLength="549"
clip-path="url(#breeze-down-line-17)">that want to bring exactly one project down. </text><text
class="breeze-down-r7" x="768.6" y="434.8" textLength="73.2"
clip-path="url(#breeze-down-line-17)">(TEXT)</text><text class="breeze-down-r5"
x="1451.8" y="434.8" textLength=" [...]
</text><text class="breeze-down-r5" x="0" y="459.2" textLength="1464"
clip-path="url(#breeze-down-line-18)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-down-r1" x="1464" y="459.2" textLength="12.2"
clip-path="url(#breeze-down-line-18)">
</text><text class="breeze-down-r5" x="0" y="483.6" textLength="24.4"
clip-path="url(#breeze-down-line-19)">╭─</text><text class="breeze-down-r5"
x="24.4" y="483.6" textLength="195.2"
clip-path="url(#breeze-down-line-19)"> Common options </text><text
class="breeze-down-r5" x="219.6" y="483.6" textLength="1220"
clip-path="url(#breeze-down-line-19)">────────────────────────────────────────────────────────────────────────────────────────────────────</text><text
class="breeze- [...]
-</text><text class="breeze-down-r5" x="0" y="508" textLength="12.2"
clip-path="url(#breeze-down-line-20)">│</text><text class="breeze-down-r4"
x="24.4" y="508" textLength="109.8"
clip-path="url(#breeze-down-line-20)">--verbose</text><text
class="breeze-down-r6" x="158.6" y="508" textLength="24.4"
clip-path="url(#breeze-down-line-20)">-v</text><text class="breeze-down-r1"
x="207.4" y="508" textLength="585.6"
clip-path="url(#breeze-down-line-20)">Print verbose information ab
[...]
-</text><text class="breeze-down-r5" x="0" y="532.4" textLength="12.2"
clip-path="url(#breeze-down-line-21)">│</text><text class="breeze-down-r4"
x="24.4" y="532.4" textLength="109.8"
clip-path="url(#breeze-down-line-21)">--dry-run</text><text
class="breeze-down-r6" x="158.6" y="532.4" textLength="24.4"
clip-path="url(#breeze-down-line-21)">-D</text><text class="breeze-down-r1"
x="207.4" y="532.4" textLength="719.8"
clip-path="url(#breeze-down-line-21)">If dry-run is set,&# [...]
-</text><text class="breeze-down-r5" x="0" y="556.8" textLength="12.2"
clip-path="url(#breeze-down-line-22)">│</text><text class="breeze-down-r4"
x="24.4" y="556.8" textLength="109.8"
clip-path="url(#breeze-down-line-22)">--help   </text><text
class="breeze-down-r6" x="158.6" y="556.8" textLength="24.4"
clip-path="url(#breeze-down-line-22)">-h</text><text class="breeze-down-r1"
x="207.4" y="556.8" textLength="329.4"
clip-path="url(#breeze-down-line-22)">Show this  [...]
-</text><text class="breeze-down-r5" x="0" y="581.2" textLength="1464"
clip-path="url(#breeze-down-line-23)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-down-r1" x="1464" y="581.2" textLength="12.2"
clip-path="url(#breeze-down-line-23)">
+</text><text class="breeze-down-r5" x="0" y="508" textLength="12.2"
clip-path="url(#breeze-down-line-20)">│</text><text class="breeze-down-r4"
x="24.4" y="508" textLength="207.4"
clip-path="url(#breeze-down-line-20)">--cleanup-pycache</text><text
class="breeze-down-r1" x="305" y="508" textLength="927.2"
clip-path="url(#breeze-down-line-20)">Additionally cleanup the Python bytecode cache volume used by Breeze shells.</text><text
class="bre [...]
+</text><text class="breeze-down-r5" x="0" y="532.4" textLength="12.2"
clip-path="url(#breeze-down-line-21)">│</text><text class="breeze-down-r4"
x="24.4" y="532.4" textLength="207.4"
clip-path="url(#breeze-down-line-21)">--verbose        </text><text
class="breeze-down-r6" x="256.2" y="532.4" textLength="24.4"
clip-path="url(#breeze-down-line-21)">-v</text><text class="breeze-down-r1"
x="305" y="532.4" textLength="585.6" clip-path="url(#breeze-down [...]
+</text><text class="breeze-down-r5" x="0" y="556.8" textLength="12.2"
clip-path="url(#breeze-down-line-22)">│</text><text class="breeze-down-r4"
x="24.4" y="556.8" textLength="207.4"
clip-path="url(#breeze-down-line-22)">--dry-run        </text><text
class="breeze-down-r6" x="256.2" y="556.8" textLength="24.4"
clip-path="url(#breeze-down-line-22)">-D</text><text class="breeze-down-r1"
x="305" y="556.8" textLength="719.8" clip-path="url(#breeze-down [...]
+</text><text class="breeze-down-r5" x="0" y="581.2" textLength="12.2"
clip-path="url(#breeze-down-line-23)">│</text><text class="breeze-down-r4"
x="24.4" y="581.2" textLength="207.4"
clip-path="url(#breeze-down-line-23)">--help           </text><text
class="breeze-down-r6" x="256.2" y="581.2" textLength="24.4"
clip-path="url(#breeze-down-line-23)">-h</text><text class="breeze-down-r1"
x="305" y="581.2" textLength="329.4" clip-path="u [...]
+</text><text class="breeze-down-r5" x="0" y="605.6" textLength="1464"
clip-path="url(#breeze-down-line-24)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-down-r1" x="1464" y="605.6" textLength="12.2"
clip-path="url(#breeze-down-line-24)">
</text>
</g>
</g>
diff --git a/dev/breeze/doc/images/output_down.txt
b/dev/breeze/doc/images/output_down.txt
index 9c1280841b6..6d8234c8ff6 100644
--- a/dev/breeze/doc/images/output_down.txt
+++ b/dev/breeze/doc/images/output_down.txt
@@ -1 +1 @@
-cde032283bb58d1cf97ee92f145f1ea8
+66a9276912f4c9c20c2743e0787a6828
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
index ce8d3293bf4..24ce9cfdfe8 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -134,6 +134,7 @@ from airflow_breeze.utils.path_utils import (
COMMON_AI_PLUGIN_PREK_HOOK,
EDGE_PLUGIN_PREK_HOOK,
FAB_AUTH_MANAGER_WWW_PREK_HOOK,
+ PYCACHE_VOLUME_NAME,
cleanup_python_generated_files,
)
from airflow_breeze.utils.platforms import get_normalized_platform
@@ -467,6 +468,7 @@ def shell(
forward_credentials=forward_credentials,
github_repository=github_repository,
include_mypy_volume=include_mypy_volume,
+ include_pycache_volume=True,
install_airflow_with_constraints=install_airflow_with_constraints,
install_airflow_python_client=install_airflow_python_client,
install_selected_providers=install_selected_providers,
@@ -711,6 +713,7 @@ def start_airflow(
force_build=force_build,
forward_credentials=forward_credentials,
github_repository=github_repository,
+ include_pycache_volume=True,
integration=integration,
install_selected_providers=install_selected_providers,
install_airflow_with_constraints=install_airflow_with_constraints,
@@ -1119,6 +1122,11 @@ def build_docs(
help="Additionally cleanup MyPy cache.",
is_flag=True,
)
[email protected](
+ "--cleanup-pycache",
+ help="Additionally cleanup the Python bytecode cache volume used by Breeze
shells.",
+ is_flag=True,
+)
@click.option(
"-b",
"--cleanup-build-cache",
@@ -1146,6 +1154,7 @@ def build_docs(
def down(
preserve_volumes: bool,
cleanup_mypy_cache: bool,
+ cleanup_pycache: bool,
cleanup_build_cache: bool,
all_projects: bool,
project_name: str | None,
@@ -1177,6 +1186,8 @@ def down(
if hook_dir.exists():
console_print(f"\n[info]Removing dedicated mypy {subdir}:
{hook_dir}\n")
shutil.rmtree(hook_dir)
+ if cleanup_pycache:
+ run_command(["docker", "volume", "rm", "--force", PYCACHE_VOLUME_NAME])
if cleanup_build_cache:
command_to_execute = ["docker", "volume", "rm", "--force",
"airflow-cache-volume"]
run_command(command_to_execute)
diff --git
a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py
b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py
index 54e1a296f2c..c0b746bb242 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py
@@ -333,6 +333,7 @@ DEVELOPER_PARAMETERS: dict[str, list[dict[str, str |
list[str]]]] = {
"options": [
"--preserve-volumes",
"--cleanup-mypy-cache",
+ "--cleanup-pycache",
"--cleanup-build-cache",
],
},
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index 51a7d020351..e70db140f2d 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -520,7 +520,9 @@ BREEZE_DEBUG_WEBSERVER_PORT = "50237"
CELERY_BROKER_URLS_MAP = {"rabbitmq": "amqp://guest:guest@rabbitmq:5672",
"redis": "redis://redis:6379/0"}
SQLITE_URL = "sqlite:////root/airflow/sqlite/airflow.db"
-PYTHONDONTWRITEBYTECODE = True
+# Bytecode cache lives outside the mounted sources so it never pollutes the
host checkout
+# and can be persisted in a docker volume across container runs (see
pycache.yml).
+PYCACHE_PREFIX_IN_CONTAINER = "/root/.cache/airflow-pycache"
PRODUCTION_IMAGE = False
# All python versions include all past python versions available in previous
branches
diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py
b/dev/breeze/src/airflow_breeze/params/shell_params.py
index 9dcb9c27019..9bcac54db23 100644
--- a/dev/breeze/src/airflow_breeze/params/shell_params.py
+++ b/dev/breeze/src/airflow_breeze/params/shell_params.py
@@ -65,6 +65,7 @@ from airflow_breeze.global_constants import (
MYSQL_HOST_PORT,
POSTGRES_BACKEND,
POSTGRES_HOST_PORT,
+ PYCACHE_PREFIX_IN_CONTAINER,
RABBITMQ_HOST_PORT,
REDIS_HOST_PORT,
SIMPLE_AUTH_MANAGER,
@@ -103,6 +104,7 @@ from airflow_breeze.utils.path_utils import (
SCRIPTS_CI_DOCKER_COMPOSE_MYPY_PATH,
SCRIPTS_CI_DOCKER_COMPOSE_PATH,
SCRIPTS_CI_DOCKER_COMPOSE_PROVIDERS_AND_TESTS_SOURCES_PATH,
+ SCRIPTS_CI_DOCKER_COMPOSE_PYCACHE_PATH,
SCRIPTS_CI_DOCKER_COMPOSE_REMOVE_SOURCES_PATH,
SCRIPTS_CI_DOCKER_COMPOSE_TESTS_SOURCES_PATH,
get_main_git_dir_for_worktree,
@@ -198,6 +200,7 @@ class ShellParams:
github_repository: str = APACHE_AIRFLOW_GITHUB_REPOSITORY
github_token: str = os.environ.get("GITHUB_TOKEN", "")
include_mypy_volume: bool = False
+ include_pycache_volume: bool = False
install_airflow_version: str = ""
install_airflow_python_client: bool = False
install_airflow_with_constraints: bool = False
@@ -430,6 +433,8 @@ class ShellParams:
compose_file_list.append(SCRIPTS_CI_DOCKER_COMPOSE_FORWARD_CREDENTIALS_PATH)
if self.include_mypy_volume:
compose_file_list.append(SCRIPTS_CI_DOCKER_COMPOSE_MYPY_PATH)
+ if self.include_pycache_volume:
+ compose_file_list.append(SCRIPTS_CI_DOCKER_COMPOSE_PYCACHE_PATH)
if self.tty == "enabled":
compose_file_list.append(SCRIPTS_CI_DOCKER_COMPOSE_ENABLE_TTY_PATH)
if "all-testable" in self.integration:
@@ -706,7 +711,15 @@ services:
_set_var(_env, "PROVIDERS_CONSTRAINTS_MODE",
self.providers_constraints_mode)
_set_var(_env, "PROVIDERS_CONSTRAINTS_REFERENCE",
self.providers_constraints_reference)
_set_var(_env, "PROVIDERS_SKIP_CONSTRAINTS",
self.providers_skip_constraints)
- _set_var(_env, "PYTHONDONTWRITEBYTECODE", "true")
+ # Both keys are always emitted (empty string == unset for Python) so
the generated
+ # compose env file keeps a stable key set across invocations. The
bytecode cache is only
+ # enabled together with the volume that persists it; other containers
stay pyc-free.
+ if self.include_pycache_volume:
+ _set_var(_env, "PYTHONDONTWRITEBYTECODE", "")
+ _set_var(_env, "PYTHONPYCACHEPREFIX", PYCACHE_PREFIX_IN_CONTAINER)
+ else:
+ _set_var(_env, "PYTHONDONTWRITEBYTECODE", "true")
+ _set_var(_env, "PYTHONPYCACHEPREFIX", "")
_set_var(_env, "PYTHON_MAJOR_MINOR_VERSION", self.python)
_set_var(_env, "QUIET", self.quiet)
_set_var(_env, "REDIS_HOST_PORT", None, REDIS_HOST_PORT)
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index 957ca69d8b0..da6a9e32626 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -39,6 +39,7 @@ from airflow_breeze.utils.path_utils import (
SCRIPTS_DOCKER_PATH,
cleanup_python_generated_files,
create_mypy_volume_if_needed,
+ create_pycache_volume_if_needed,
get_main_git_dir_for_worktree,
)
from airflow_breeze.utils.shared_options import get_verbose
@@ -1110,6 +1111,8 @@ def enter_shell(
bring_compose_project_down(preserve_volumes=False,
shell_params=shell_params)
if shell_params.include_mypy_volume:
create_mypy_volume_if_needed()
+ if shell_params.include_pycache_volume:
+ create_pycache_volume_if_needed()
shell_params.print_badge_info()
cmd = ["docker", "compose"]
if shell_params.quiet:
diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py
b/dev/breeze/src/airflow_breeze/utils/path_utils.py
index eef8f73226a..414fca44116 100644
--- a/dev/breeze/src/airflow_breeze/utils/path_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py
@@ -524,6 +524,7 @@ SCRIPTS_CI_DOCKER_COMPOSE_LOCAL_ALL_SOURCES_PATH =
SCRIPTS_CI_DOCKER_COMPOSE_PAT
SCRIPTS_CI_DOCKER_COMPOSE_LOCAL_YAML_PATH = SCRIPTS_CI_DOCKER_COMPOSE_PATH /
"local.yml"
SCRIPTS_CI_DOCKER_COMPOSE_MOUNT_UI_DIST_PATH = SCRIPTS_CI_DOCKER_COMPOSE_PATH
/ "mount-ui-dist.yml"
SCRIPTS_CI_DOCKER_COMPOSE_MYPY_PATH = SCRIPTS_CI_DOCKER_COMPOSE_PATH /
"mypy.yml"
+SCRIPTS_CI_DOCKER_COMPOSE_PYCACHE_PATH = SCRIPTS_CI_DOCKER_COMPOSE_PATH /
"pycache.yml"
SCRIPTS_CI_DOCKER_COMPOSE_PROVIDERS_AND_TESTS_SOURCES_PATH = (
SCRIPTS_CI_DOCKER_COMPOSE_PATH / "providers-and-tests-sources.yml"
)
@@ -542,6 +543,9 @@ BREEZE_IMAGES_PATH = BREEZE_DOC_PATH / "images"
AIRFLOW_HOME_PATH = Path(os.environ.get("AIRFLOW_HOME", Path.home() /
"airflow"))
+PYCACHE_VOLUME_NAME = "airflow-pycache-volume"
+
+
def create_volume_if_missing(volume_name: str):
from airflow_breeze.utils.run_utils import run_command
@@ -569,6 +573,10 @@ def create_mypy_volume_if_needed():
create_volume_if_missing("mypy-cache-volume")
+def create_pycache_volume_if_needed():
+ create_volume_if_missing(PYCACHE_VOLUME_NAME)
+
+
def create_directories_and_files() -> None:
"""
Creates all directories and files that are needed for Breeze to work via
docker-compose.
diff --git a/dev/breeze/tests/test_docker_command_utils.py
b/dev/breeze/tests/test_docker_command_utils.py
index 555cf897b3c..a0ef5bffd73 100644
--- a/dev/breeze/tests/test_docker_command_utils.py
+++ b/dev/breeze/tests/test_docker_command_utils.py
@@ -467,6 +467,7 @@ def _shell_params_for_openlineage(
shell_params.use_airflow_version = None
shell_params.restart = False
shell_params.include_mypy_volume = False
+ shell_params.include_pycache_volume = False
shell_params.quiet = True
shell_params.project_name = None
shell_params.tty = "disabled"
diff --git a/dev/breeze/tests/test_shell_params.py
b/dev/breeze/tests/test_shell_params.py
index 8dd642727e5..67980076b5c 100644
--- a/dev/breeze/tests/test_shell_params.py
+++ b/dev/breeze/tests/test_shell_params.py
@@ -17,6 +17,7 @@
from __future__ import annotations
+import os
from unittest.mock import patch
import pytest
@@ -24,10 +25,12 @@ import yaml
from rich.console import Console
from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
+from airflow_breeze.global_constants import PYCACHE_PREFIX_IN_CONTAINER
from airflow_breeze.params.shell_params import ShellParams
from airflow_breeze.utils.path_utils import (
SCRIPTS_CI_DOCKER_COMPOSE_BASE_PATH,
SCRIPTS_CI_DOCKER_COMPOSE_PATH,
+ SCRIPTS_CI_DOCKER_COMPOSE_PYCACHE_PATH,
)
console = Console(width=400, color_system="standard")
@@ -259,6 +262,26 @@ def
test_pythonwarnings_is_forwarded_by_the_compose_base_file():
assert "PYTHONWARNINGS" in
base_compose_file["services"]["airflow"]["environment"]
[email protected](
+ ("include_pycache_volume", "expected_prefix", "expected_dont_write"),
+ [(True, PYCACHE_PREFIX_IN_CONTAINER, ""), (False, "", "true")],
+)
+def test_bytecode_cache_is_enabled_only_together_with_its_volume(
+ include_pycache_volume: bool, expected_prefix: str, expected_dont_write:
str
+):
+ env_vars =
ShellParams(include_pycache_volume=include_pycache_volume).env_variables_for_docker_commands
+ assert env_vars["PYTHONPYCACHEPREFIX"] == expected_prefix
+ assert env_vars["PYTHONDONTWRITEBYTECODE"] == expected_dont_write
+
+
[email protected](("include_pycache_volume", "expected_count"), [(True,
1), (False, 0)])
+def test_pycache_volume_compose_file_is_included_only_when_requested(
+ include_pycache_volume: bool, expected_count: int
+):
+ compose_files =
ShellParams(include_pycache_volume=include_pycache_volume).compose_file.split(os.pathsep)
+ assert compose_files.count(str(SCRIPTS_CI_DOCKER_COMPOSE_PYCACHE_PATH)) ==
expected_count
+
+
def test_include_mypy_volume_adds_mypy_compose_file():
compose_files =
ShellParams(include_mypy_volume=True).compose_file.split(":")
assert str(SCRIPTS_CI_DOCKER_COMPOSE_PATH / "mypy.yml") in compose_files
diff --git a/scripts/ci/docker-compose/pycache.yml
b/scripts/ci/docker-compose/pycache.yml
new file mode 100644
index 00000000000..69683377422
--- /dev/null
+++ b/scripts/ci/docker-compose/pycache.yml
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+services:
+ airflow:
+ volumes:
+ - type: volume
+ source: airflow-pycache-volume
+ target: ${PYTHONPYCACHEPREFIX}
+volumes:
+ airflow-pycache-volume:
+ external: true
