This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 5387c0c761 Fix installing older airflow versions in Breeze with
openssl fix (#33171)
5387c0c761 is described below
commit 5387c0c761fe161f28cee70a46009e1e67286679
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Aug 7 21:28:40 2023 +0200
Fix installing older airflow versions in Breeze with openssl fix (#33171)
The dnspython and Flask Application Builder have a weird transitive
dependency to openssl. It does not require it - it is an optional
dependnency of email validator that uses dnspython and the dnspython
uses pyopenssl when installed. This is all fine. But when opendns
is used with older FAB < 4.1.4) and NEWER opensl (>=23.0), just
creating the FAB applicatio in flask causes an exception than
there is a missing parameter in openssl.
Since openssl is NOT declared as required dependency in neither FAB
nor dnspython, installing airflow with constraints while
having a newer version of pyopenssl does not downgrade pyopenssl
it is just mentioned in a conflicting message resulting from conflicts
with other packages already installed (such as cryptography).
This caused a problem when installin older airflow version in breeze
with --use-airflow-version switch.
Also - during checking this issue, it turned out that we were not
- by default - using the constraints for older version when installing
it in breeze (you could do that by explicitly specifying the
constraints). This was because when you used anoter way to specify
versio (commit hash, tag) it is quite complex to figure out which
constraints to use. However we can automatically derive constraints
when you specify version, or branch, which are the most common
scenarios.
This PR fixes the `--use-airflow-version` case for Airflow 2.4 and below
by:
* automatically deriving the right constraints when version or branch
are used in `--use-airflow-version`
* adding pyopenssl specifically as dependency when
--use-airflow-version switch is used.
* also a small quality of life improvment - when airflow db migrate
fails (because old versio of airflow does not support it) - we inform
the user that it happened and that we are attempting to run the
legacy airflow db init instead.
---
.../airflow_breeze/commands/developer_commands.py | 43 +++++++++++++++++++---
scripts/in_container/_in_container_utils.sh | 13 ++++++-
scripts/in_container/check_environment.sh | 3 ++
3 files changed, 52 insertions(+), 7 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
index 4f0d14a174..f29584bb32 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -17,6 +17,7 @@
from __future__ import annotations
import os
+import re
import shutil
import sys
import threading
@@ -26,6 +27,7 @@ from typing import Iterable
import click
+from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
from airflow_breeze.commands.ci_image_commands import
rebuild_or_pull_ci_image_if_needed
from airflow_breeze.commands.main_command import main
from airflow_breeze.global_constants import (
@@ -96,12 +98,26 @@ from airflow_breeze.utils.run_utils import (
from airflow_breeze.utils.shared_options import get_dry_run, get_verbose,
set_forced_answer
from airflow_breeze.utils.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET,
CHEATSHEET_STYLE
-# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-# Make sure that whatever you add here as an option is also
-# Added in the "main" command in breeze.py. The min command above
-# Is used for a shorthand of shell and except the extra
-# Args it should have the same parameters.
-# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+def _determine_constraint_branch_used(airflow_constraints_reference: str,
use_airflow_version: str | None):
+ """
+ Determine which constraints reference to use.
+
+ When use-airflow-version is branch or version, we derive the constraints
branch from it, unless
+ someone specified the constraints branch explicitly.
+
+ :param airflow_constraints_reference: the constraint reference specified
(or default)
+ :param use_airflow_version: which airflow version we are installing
+ :return: the actual constraints reference to use
+ """
+ if (
+ use_airflow_version
+ and airflow_constraints_reference == DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
+ and re.match(r"[0-9]+\.[0-9]+\.[0-9]+[0-9a-z\.]*|main|v[0-9]_.*",
use_airflow_version)
+ ):
+ get_console().print(f"[info]Using constraints {use_airflow_version} -
matching airflow version used.")
+ return use_airflow_version
+ return airflow_constraints_reference
class TimerThread(threading.Thread):
@@ -116,6 +132,14 @@ class TimerThread(threading.Thread):
os.killpg(os.getpgid(0), SIGTERM)
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+# Make sure that whatever you add here as an option is also
+# Added in the "main" command in breeze.py. The min command above
+# Is used for a shorthand of shell and except the extra
+# Args it should have the same parameters.
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+
@main.command()
@option_python
@option_platform_single
@@ -181,6 +205,9 @@ def shell(
if max_time:
TimerThread(max_time=max_time).start()
set_forced_answer("yes")
+ airflow_constraints_reference = _determine_constraint_branch_used(
+ airflow_constraints_reference, use_airflow_version
+ )
result = enter_shell(
python=python,
github_repository=github_repository,
@@ -292,6 +319,10 @@ def start_airflow(
skip_asset_compilation = True
if use_airflow_version is None and not skip_asset_compilation:
run_compile_www_assets(dev=dev_mode, run_in_background=True)
+ airflow_constraints_reference = _determine_constraint_branch_used(
+ airflow_constraints_reference, use_airflow_version
+ )
+
result = enter_shell(
python=python,
github_repository=github_repository,
diff --git a/scripts/in_container/_in_container_utils.sh
b/scripts/in_container/_in_container_utils.sh
index 8d4996a2f5..92d6ccf250 100644
--- a/scripts/in_container/_in_container_utils.sh
+++ b/scripts/in_container/_in_container_utils.sh
@@ -262,7 +262,18 @@ function install_released_airflow_version() {
if [[ ${constraints_reference} == "none" ]]; then
pip install "${airflow_package}${extras}"
else
- pip install "apache-airflow${BRACKETED_AIRFLOW_EXTRAS}==${version}" \
+ local dependency_fix=""
+ # The pyopenssl is needed to downgrade pyopenssl for older airflow
versions when using constraints
+ # Flask app builder has an optional pyopenssl transitive dependency,
that causes import error when
+ # Pyopenssl is installed in a wrong version for Flask App Builder 4.1
and older. Adding PyOpenSSL
+ # directly as the dependency, forces downgrading of pyopenssl to the
right version. Our constraint
+ # version has it pinned to the right version, but since it is not
directly required, it is not
+ # downgraded when installing airflow and it is already installed in a
newer version
+ if [[ ${USE_AIRFLOW_VERSION=} != "" ]]; then
+ dependency_fix="pyopenssl"
+ fi
+
+ pip install "apache-airflow${BRACKETED_AIRFLOW_EXTRAS}==${version}"
${dependency_fix} \
--constraint
"https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/constraints-${version}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt";
fi
}
diff --git a/scripts/in_container/check_environment.sh
b/scripts/in_container/check_environment.sh
index 67d4f1814e..9bdb9fcc15 100755
--- a/scripts/in_container/check_environment.sh
+++ b/scripts/in_container/check_environment.sh
@@ -123,6 +123,9 @@ function startairflow_if_requested() {
airflow connections create-default-connections
fi
else
+ echo "${COLOR_YELLOW}Failed to run 'airflow db
migrate'.${COLOR_RESET}"
+ echo "${COLOR_BLUE}This could be because you are installing old
airflow version${COLOR_RESET}"
+ echo "${COLOR_BLUE}Attempting to run deprecated 'airflow db init'
instead.${COLOR_RESET}"
# For Airflow versions that do not support db migrate, we should
run airflow db init and
# set the removed AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS
AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS} airflow
db init
