This is an automated email from the ASF dual-hosted git repository. kaxilnaik pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 91948939ebbcc3e302b4d15eac8a41b1dd913cb6 Author: Ash Berlin-Taylor <ash_git...@firemirror.com> AuthorDate: Fri Nov 20 21:21:13 2020 +0000 Add upgrade check rule to ensure on "latest" versions (#12514) This checks against PyPI to make sure this is run with the latest non-preview release of apache-airflow-upgrade-check, and the latest 1.10.x of apache-airflow (cherry picked from commit d7ace0267c34bd5520d321a495399710c1c49cd1) --- airflow/upgrade/rules/__init__.py | 2 +- airflow/upgrade/rules/aaa_airflow_version_check.py | 87 ++++++++++++++++++++++ setup.py | 2 + .../rules/test_aaa_airflow_version_check.py | 75 +++++++++++++++++++ 4 files changed, 165 insertions(+), 1 deletion(-) diff --git a/airflow/upgrade/rules/__init__.py b/airflow/upgrade/rules/__init__.py index 4735c7f..97d0160 100644 --- a/airflow/upgrade/rules/__init__.py +++ b/airflow/upgrade/rules/__init__.py @@ -21,7 +21,7 @@ def get_rules(): """Automatically discover all rules""" rule_classes = [] path = os.path.dirname(os.path.abspath(__file__)) - for file in os.listdir(path): + for file in sorted(os.listdir(path)): if not file.endswith(".py") or file in ("__init__.py", "base_rule.py"): continue py_file = file[:-3] diff --git a/airflow/upgrade/rules/aaa_airflow_version_check.py b/airflow/upgrade/rules/aaa_airflow_version_check.py new file mode 100644 index 0000000..ad84eb4 --- /dev/null +++ b/airflow/upgrade/rules/aaa_airflow_version_check.py @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This module starts with `aaa_` so that it is sorted first alphabetically, but is still a valid python module +# name (starting with digitis is not valid) + +from __future__ import absolute_import + +from packaging.version import Version +import requests + +from airflow.upgrade.rules.base_rule import BaseRule + +try: + import importlib.metadata as importlib_metadata +except ImportError: + import importlib_metadata + + +class VersionCheckRule(BaseRule): + + title = "Check for latest versions of apache-airflow and checker" + + description = """\ +Check that the latest version of apache-airflow-upgrade-check is installed, and +that you are on the latest 1.10.x release of apache-airflow.""" + + def pypi_releases(self, distname): + """ + Get all the non-dev releases of a dist from PyPI + """ + + resp = requests.get("https://pypi.org/pypi/{}/json".format(distname)) + resp.raise_for_status() + + for rel_string in resp.json()["releases"].keys(): + ver = Version(rel_string) + if ver.is_devrelease or ver.is_prerelease: + continue + yield ver + + def check(self): + + current_airflow_version = Version(__import__("airflow").__version__) + try: + upgrade_check_ver = Version( + importlib_metadata.distribution("apache-airflow-upgrade-check").version, + ) + except importlib_metadata.PackageNotFoundError: + upgrade_check_ver = Version("0.0.0") + + try: + latest_airflow_v1_release = sorted( + filter(lambda v: v.major == 1, self.pypi_releases("apache-airflow")) + )[-1] + + if current_airflow_version < latest_airflow_v1_release: + yield ( + "There is a more recent version of apache-airflow. Please upgrade to {} and re-run this" + " script" + ).format(latest_airflow_v1_release) + + latest_upgrade_check_release = sorted( + self.pypi_releases("apache-airflow-upgrade-check") + )[-1] + + if upgrade_check_ver < latest_upgrade_check_release: + yield ( + "There is a more recent version of apache-airflow-upgrade-check. Please upgrade to {}" + " and re-run this script" + ).format(latest_upgrade_check_release) + except Exception as e: + yield "Unable to go ask PyPI.org for latest release information: " + str(e) diff --git a/setup.py b/setup.py index 30917d2..9617ac7 100644 --- a/setup.py +++ b/setup.py @@ -603,6 +603,7 @@ INSTALL_REQUIREMENTS = [ 'future>=0.16.0, <0.19', 'graphviz>=0.12', 'gunicorn>=19.5.0, <21.0', + 'importlib-metadata~=2.0; python_version<"3.8"', 'iso8601>=0.1.12', 'jinja2>=2.10.1, <2.12.0', 'json-merge-patch==0.2', @@ -611,6 +612,7 @@ INSTALL_REQUIREMENTS = [ 'markdown>=2.5.2, <3.0', 'marshmallow-sqlalchemy>=0.16.1, <0.24.0;python_version>="3.6"', 'marshmallow-sqlalchemy>=0.16.1, <0.19.0;python_version<"3.6"', + 'packaging', 'pandas>=0.17.1, <2.0', 'pendulum==1.4.4', 'pep562~=1.0;python_version<"3.7"', diff --git a/tests/upgrade/rules/test_aaa_airflow_version_check.py b/tests/upgrade/rules/test_aaa_airflow_version_check.py new file mode 100644 index 0000000..1c4fce0 --- /dev/null +++ b/tests/upgrade/rules/test_aaa_airflow_version_check.py @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from airflow.upgrade.rules.aaa_airflow_version_check import VersionCheckRule + + +class TestVersionCheckRule: + def test_new_airflow_version_available(self, requests_mock): + requests_mock.get( + "https://pypi.org/pypi/apache-airflow/json", + json={ + "releases": { + # The values here don't matter + "1.10.22b1": None, + "1.10.20": None, + "1.10.21": None, + } + }, + ) + requests_mock.get( + "https://pypi.org/pypi/apache-airflow-upgrade-check/json", + json={ + "releases": { + "0.0.0": None, + } + }, + ) + + results = list(VersionCheckRule().check()) + assert len(results) == 1, results + + msg = results[0] + assert " 1.10.21 " in msg + assert "apache-airflow" in msg + assert "apache-airflow-upgrade-check" not in msg + + def test_new_airflow_upgrade_check_version_available(self, requests_mock): + requests_mock.get( + "https://pypi.org/pypi/apache-airflow/json", + json={ + "releases": { + # The values here don't matter + "1.10.0": None, + } + }, + ) + requests_mock.get( + "https://pypi.org/pypi/apache-airflow-upgrade-check/json", + json={ + "releases": { + "99.0.0": None, + } + }, + ) + + results = list(VersionCheckRule().check()) + assert len(results) == 1, results + + msg = results[0] + assert " 99.0.0 " in msg + assert "apache-airflow-upgrade-check" in msg