potiuk commented on code in PR #49799:
URL: https://github.com/apache/airflow/pull/49799#discussion_r2061032500
##########
scripts/in_container/run_generate_constraints.py:
##########
@@ -304,76 +312,21 @@ def generate_constraints_pypi_providers(config_params:
ConfigParams) -> None:
providers are used by our users to install Airflow in reproducible way.
:return:
"""
- all_provider_distributions =
get_all_active_provider_distributions(python_version=config_params.python)
- chicken_egg_prefixes = []
- packages_to_install = []
- console.print("[bright_blue]Installing Airflow with PyPI providers with
eager upgrade")
- if config_params.chicken_egg_providers:
- for chicken_egg_provider in
config_params.chicken_egg_providers.split(" "):
-
chicken_egg_prefixes.append(f"apache-airflow-providers-{chicken_egg_provider.replace('.',
'-')}")
- console.print(
- f"[bright_blue]Checking if {chicken_egg_prefixes} are available in
local dist folder "
- f"as chicken egg providers)"
- )
- for provider_package in all_provider_distributions:
- if config_params.chicken_egg_providers and
provider_package.startswith(tuple(chicken_egg_prefixes)):
- glob_pattern = f"{provider_package.replace('-', '_')}-*.whl"
- console.print(
- f"[bright_blue]Checking if {provider_package} is available in
local dist folder "
- f"with {glob_pattern} pattern"
- )
- files = AIRFLOW_DIST_PATH.glob(glob_pattern)
- for file in files:
- console.print(
- f"[yellow]Installing {file.name} from local dist folder as
it is a chicken egg provider"
- )
- packages_to_install.append(f"{provider_package} @
file://{file.as_posix()}")
- break
- else:
- console.print(
- f"[yellow]Skipping {provider_package} as it is not found
in dist folder to install."
- )
- # Skip checking if chicken egg provider is available in PyPI - it
does not have to be there
- continue
- console.print(f"[bright_blue]Checking if {provider_package} is
available in PyPI: ... ", end="")
- r = requests.get(f"https://pypi.org/pypi/{provider_package}/json";,
timeout=60)
- if r.status_code == 200:
- version = r.json()["info"]["version"]
- console.print("[green]OK")
- # TODO: In the future we might make it a bit more sophisticated
and allow to install
- # earlier versions of providers here - but for now we just install
the latest version
- # But in practice latest version does not have to be supported by
current version of airflow
- # This should be a very rare case though - such provider
distributions should only be released
- # in pre-release mode
- packages_to_install.append(f"{provider_package}=={version}")
- else:
- console.print("[yellow]NOK. Skipping.")
- find_airflow_distributions = AIRFLOW_DIST_PATH.glob("apache_airflow-*.whl")
- airflow_install = "."
- if find_airflow_distributions:
- airflow_install = next(find_airflow_distributions).as_posix()
- airflow_core_install = "./airflow-core[all]"
- find_airflow_core_distributions =
AIRFLOW_DIST_PATH.glob("apache_airflow_core-*.whl")
- if find_airflow_core_distributions:
- airflow_core_install =
next(find_airflow_core_distributions).as_posix() + "[all]"
- airflow_task_sdk_install = "./airflow-task-sdk"
- find_airflow_task_sdk_distribution =
AIRFLOW_DIST_PATH.glob("apache_airflow_task_sdk-*.whl")
- if find_airflow_task_sdk_distribution:
- airflow_task_sdk_install =
next(find_airflow_task_sdk_distribution).as_posix()
run_command(
cmd=[
"uv",
"pip",
"install",
"--no-sources",
- airflow_install,
- airflow_core_install,
- airflow_task_sdk_install,
+ "apache-airflow[all]",
+ "apache-airflow-core[all]",
+ "apache-airflow-task-sdk",
"./airflow-ctl",
- "--reinstall", # We need to pull the provider distributions from
PyPI - not use the local ones
- *packages_to_install,
+ "--reinstall", # We need to pull the provider distributions from
PyPI or dist, not the local ones
"--resolution",
"highest",
+ "--find-links",
Review Comment:
We can now simply rely on `uv pip` finding newer versions of providers in
the `dist` folder where we prepared all the non-released providers yet.
Previously we had to decide which packages we build and manually choose those
packages because we were not sure if they were released yet or not.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
