pierrejeambrun commented on code in PR #35086:
URL: https://github.com/apache/airflow/pull/35086#discussion_r1367924567
##########
dev/breeze/src/airflow_breeze/commands/sbom_commands.py:
##########
@@ -291,25 +291,74 @@ def generate_providers_requirements(
with open(PROVIDER_METADATA_JSON_FILE_PATH) as f:
provider_metadata = json.load(f)
providers_info = [
- (p_id, p_version, info["associated_airflow_version"])
- for (p_id, p_versions) in provider_metadata.items()
- for (p_version, info) in p_versions.items()
+ (
+ provider_id,
+ provider_version,
+ python_version,
+ info["associated_airflow_version"],
+ )
+ for (provider_id, provider_versions) in
provider_metadata.items()
+ for (provider_version, info) in provider_versions.items()
+ for python_version in
AIRFLOW_PYTHON_COMPATIBILITY_MATRIX[info["associated_airflow_version"]]
+ if python_version in python_versions
]
else:
if airflow_version is None:
airflow_version = get_active_airflow_versions(confirm=False)[-1]
get_console().print(f"[info]Using {airflow_version} as airflow
version")
- providers_info = [(provider_id, None, airflow_version) for provider_id
in provider_ids]
+ providers_info = [
+ (provider_id, None, python_version, airflow_version)
+ for provider_id in provider_ids
+ for python_version in
AIRFLOW_PYTHON_COMPATIBILITY_MATRIX[airflow_version]
+ if python_version in python_versions
+ ]
- build_all_airflow_versions_base_image(python_version=python)
+ if run_in_parallel:
+ parallelism = min(parallelism, len(python_versions))
+ get_console().print(f"[info]Running {len(python_versions)} jobs in
parallel")
+ with ci_group(f"Building all airflow base images for python:
{python_versions}"):
+ all_params = [
+ f"Building all airflow base image for python: {python_version}"
+ for python_version in python_versions
+ ]
+ with run_with_pool(
+ parallelism=parallelism,
+ all_params=all_params,
+ debug_resources=debug_resources,
+ progress_matcher=ShowLastLineProgressMatcher(),
+ ) as (pool, outputs):
+ results = [
+ pool.apply_async(
+ build_all_airflow_versions_base_image,
+ kwds={
+ "python_version": python_version,
+ "confirm": False,
+ },
Review Comment:
Done, I updated with output parameter and used the
`DockerBuildxProgressMatcher`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
