This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ba28334621 Fix local k8s test venv installation to accomodate for
local changes (#35191)
ba28334621 is described below
commit ba283346215278404912baca4854d78b64e1e7d2
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Thu Oct 26 11:05:01 2023 +0200
Fix local k8s test venv installation to accomodate for local changes
(#35191)
The #35099 switched installation of the local venv for Airflow to
released packages, in order to accomodate for case from #34729 where
we locally install a new provider that is preinstalled but has not
yet been released.
This however has the side-effect - some k8s tests are using K8S Pod
operator to run the tests and if this operator has been changed, the
local - modified - version of it is not going to be used for tests.
Another side effect is that in case of a new installation, when the
constraint installation does not work (for example constraints in
main are conflicting with released airflow), `pip` starts to
backtrack - and for conflicting requirements it produces strange
results (for example it attempts to install airflow 1.10 and fails
due to "slugify" dependency.
The previous version of it had another problem - once installed,
it had not changed, so in case someone used breeze to run k8s tests
locally and iterated over changes in K8SPod Operator, only the
version from the moment the k8s environment was installed was used.
Both cases are now handled better:
* INSTALL_PROVIDERS_FROM_SOURCES is used as env variable to make
sure new providers (including preinstalled providers) are found
in Airfow sources, not in PyPI
* The "." is used back in order to install Airflow but also the
-e (editable) installation is used for it - so that any changes
to local version of k8s are used. We are not using constraints
for installation.
* Dry run and verbose mode of installing the venv are improved
a bit - they show better what's going on (and dry_run does
not interact with the installation in unexpected ways - deleting
the installed venv without recreating it).
We already handled a change that k8s test environment has been
reinstalled after the requirements file changed and caching in CI
includes the hash of the requirements file as content - so we do not
need to handle reinstallation of the venv or caching in CI. The
venv should be appropriately reinstalled as needed.
---
.github/workflows/ci.yml | 2 +-
.../src/airflow_breeze/utils/kubernetes_utils.py | 47 +++++++++++-----------
scripts/ci/kubernetes/k8s_requirements.txt | 2 +-
3 files changed, 26 insertions(+), 25 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3dd00e2094..28e0a41372 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1683,7 +1683,7 @@ jobs:
KUBERNETES_VERSIONS:
${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
EXECUTOR: ${{matrix.executor}}
USE_STANDARD_NAMING: ${{matrix.use-standard-naming}}
- VERBOSE: false
+ VERBOSE: true
- name: Upload KinD logs on failure
${{needs.build-info.outputs.kubernetes-combos-list-as-string}}
uses: actions/upload-artifact@v3
if: failure() || cancelled()
diff --git a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
index 3bed798c22..7c2e6a7230 100644
--- a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
@@ -32,13 +32,12 @@ from time import sleep
from typing import Any, NamedTuple
from urllib import request
-from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
from airflow_breeze.global_constants import ALLOWED_ARCHITECTURES,
HELM_VERSION, KIND_VERSION, PIP_VERSION
from airflow_breeze.utils.console import Output, get_console
from airflow_breeze.utils.host_info_utils import Architecture,
get_host_architecture, get_host_os
from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT,
BUILD_CACHE_DIR
from airflow_breeze.utils.run_utils import RunCommandResult, run_command
-from airflow_breeze.utils.shared_options import get_dry_run
+from airflow_breeze.utils.shared_options import get_dry_run, get_verbose
K8S_ENV_PATH = BUILD_CACHE_DIR / ".k8s-env"
K8S_CLUSTERS_PATH = BUILD_CACHE_DIR / ".k8s-clusters"
@@ -270,7 +269,7 @@ def make_sure_kubernetes_tools_are_installed():
def _requirements_changed() -> bool:
if not CACHED_K8S_REQUIREMENTS.exists():
get_console().print(
- f"\n[warning]The K8S venv in {K8S_ENV_PATH}. has never been
created. Installing it.\n"
+ f"\n[warning]The K8S venv in {K8S_ENV_PATH} has never been
created. Installing it.\n"
)
return True
requirements_file_content = K8S_REQUIREMENTS.read_text()
@@ -284,7 +283,7 @@ def _requirements_changed() -> bool:
return False
-def _install_packages_in_k8s_virtualenv(with_constraints: bool):
+def _install_packages_in_k8s_virtualenv():
install_command = [
str(PYTHON_BIN_PATH),
"-m",
@@ -293,20 +292,19 @@ def _install_packages_in_k8s_virtualenv(with_constraints:
bool):
"-r",
str(K8S_REQUIREMENTS.resolve()),
]
- if with_constraints:
- install_command.extend(
- [
- "--constraint",
-
f"https://raw.githubusercontent.com/apache/airflow/{DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH}/";
-
f"constraints-{sys.version_info.major}.{sys.version_info.minor}.txt",
- ]
- )
- install_packages_result = run_command(install_command, check=False,
capture_output=True, text=True)
+ env = os.environ.copy()
+ env["INSTALL_PROVIDERS_FROM_SOURCES"] = "true"
+ capture_output = True
+ if get_verbose():
+ capture_output = False
+ install_packages_result = run_command(
+ install_command, check=False, capture_output=capture_output,
text=True, env=env
+ )
if install_packages_result.returncode != 0:
- get_console().print(
- f"[error]Error when installing packages from :
{K8S_REQUIREMENTS.resolve()}[/]\n"
-
f"{install_packages_result.stdout}\n{install_packages_result.stderr}"
- )
+ get_console().print(f"[error]Error when installing packages from :
{K8S_REQUIREMENTS.resolve()}[/]\n")
+ if not get_verbose():
+ get_console().print(install_packages_result.stdout)
+ get_console().print(install_packages_result.stderr)
return install_packages_result
@@ -328,7 +326,10 @@ def create_virtualenv(force_venv_setup: bool) ->
RunCommandResult:
get_console().print(f"[info]Forcing initializing K8S virtualenv in
{K8S_ENV_PATH}")
else:
get_console().print(f"[info]Initializing K8S virtualenv in
{K8S_ENV_PATH}")
- shutil.rmtree(K8S_ENV_PATH, ignore_errors=True)
+ if get_dry_run():
+ get_console().print(f"[info]Dry run - would be removing
{K8S_ENV_PATH}")
+ else:
+ shutil.rmtree(K8S_ENV_PATH, ignore_errors=True)
venv_command_result = run_command(
[sys.executable, "-m", "venv", str(K8S_ENV_PATH)],
check=False,
@@ -354,12 +355,12 @@ def create_virtualenv(force_venv_setup: bool) ->
RunCommandResult:
return pip_reinstall_result
get_console().print(f"[info]Installing necessary packages in
{K8S_ENV_PATH}")
- install_packages_result =
_install_packages_in_k8s_virtualenv(with_constraints=True)
- if install_packages_result.returncode != 0:
- # if the first installation fails, attempt to install it without
constraints
- install_packages_result =
_install_packages_in_k8s_virtualenv(with_constraints=False)
+ install_packages_result = _install_packages_in_k8s_virtualenv()
if install_packages_result.returncode == 0:
- CACHED_K8S_REQUIREMENTS.write_text(K8S_REQUIREMENTS.read_text())
+ if get_dry_run():
+ get_console().print(f"[info]Dry run - would be saving
{K8S_REQUIREMENTS} to cache")
+ else:
+ CACHED_K8S_REQUIREMENTS.write_text(K8S_REQUIREMENTS.read_text())
return install_packages_result
diff --git a/scripts/ci/kubernetes/k8s_requirements.txt
b/scripts/ci/kubernetes/k8s_requirements.txt
index 0922c62b2c..0f6cc2fd38 100644
--- a/scripts/ci/kubernetes/k8s_requirements.txt
+++ b/scripts/ci/kubernetes/k8s_requirements.txt
@@ -1,4 +1,4 @@
-apache-airflow[cncf.kubernetes]
+-e .[kubernetes]
pytest
pytest-cov
pytest-instafail
