FrankYang0529 commented on code in PR #76225:
URL: https://github.com/apache/airflow/pull/76225#discussion_r4463494796
##########
dev/breeze/src/airflow_breeze/commands/release_management_commands.py:
##########
@@ -5374,7 +5392,335 @@ def mirror_base_images(python: str | None):
if github_output:
with open(github_output, "a") as output_file:
output_file.write(f"moved-past-pins={'true' if moved else
'false'}\n")
+
output_file.write(f"os-deps-base-image-digests={json.dumps(os_deps_base_image_digests)}\n")
if failed:
console_print(f"[error]Failed to mirror: {', '.join(failed)}[/]")
sys.exit(1)
console_print("[success]All base images mirrored[/]")
+
+
+OS_DEPS_IMAGE_PLATFORMS = ["linux/amd64", "linux/arm64"]
+INSTALL_OS_DEPENDENCIES_SCRIPT_PATH = AIRFLOW_ROOT_PATH / "scripts" / "docker"
/ "install_os_dependencies.sh"
+
+
+def extract_dockerfile_stage(dockerfile_content: str, target: str) -> str:
+ """Return the text of one stage of a Dockerfile, from its FROM line to the
next stage."""
+ match = re.search(
+ rf"^FROM \S+ as {re.escape(target)}\n.*?(?=^FROM |\Z)",
dockerfile_content, re.MULTILINE | re.DOTALL
+ )
+ if not match:
+ raise ValueError(f"The Dockerfile has no stage named {target!r}")
+ return match.group(0)
+
+
+def get_os_deps_image_build_args(
+ image_type: OsDepsImageType, python_tag: str, python_digest: str
+) -> dict[str, str]:
+ """
+ Return the build args of an OS-dependencies image built on a mirrored base
image tag.
+
+ They must give the installation the same environment the CI and PROD
builds give it, or the
+ marker install_os_dependencies.sh writes would not match there and the
installation would run again.
+ Those builds pass the Python patchlevel they are pinned to - the one in
the base image tag - and
+ the database clients of their default parameters, which the images install
too.
+ """
+ build_args = {
+ "BASE_IMAGE":
f"{HARDENED_PYTHON_IMAGE_MIRROR}:{python_tag}@{python_digest}",
+ "AIRFLOW_PYTHON_VERSION": python_tag.split("-", 1)[0],
+ }
+ if image_type.dockerfile == "Dockerfile":
+ prod_params = BuildProdParams()
+ build_args.update(
+ {
+ "AIRFLOW_IMAGE_FLAVOR": "hardened",
+ "INSTALL_MYSQL_CLIENT": prod_params.install_mysql_client,
+ "INSTALL_MYSQL_CLIENT_TYPE":
prod_params.install_mysql_client_type,
+ "INSTALL_MSSQL_CLIENT": prod_params.install_mssql_client,
+ "INSTALL_POSTGRES_CLIENT": prod_params.install_postgres_client,
+ }
+ )
+ else:
+ build_args["INSTALL_MYSQL_CLIENT_TYPE"] =
BuildCiParams().install_mysql_client_type
+ return build_args
+
+
+OS_DEPS_DATABASE_CLIENT_SCRIPTS = ("common.sh", "install_mysql.sh",
"install_mssql.sh", "install_postgres.sh")
Review Comment:
The list of files that the marker hashes appears in three places: the `for
file in` loop in `install_os_dependencies.sh`,
`OS_DEPS_DATABASE_CLIENT_SCRIPTS` here, and
`FileGroupForCi.OS_DEPENDENCIES_FILES` in `selective_checks.py`.
Suppose a script is in the `for file in` loop but missing from
`OS_DEPS_DATABASE_CLIENT_SCRIPTS`. A change to that script then no longer
changes the content tag of the OS-dependencies image, so the daily build treats
the image as already built. Meanwhile the marker no longer matches in any CI or
PROD build, so each build reinstalls from the live repositories.
`test_marker_covers_the_database_clients` already parses the marker block.
Could that test, or a new one, also compare the marker's file list with
`OS_DEPS_DATABASE_CLIENT_SCRIPTS` and `OS_DEPENDENCIES_FILES`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
