This is an automated email from the ASF dual-hosted git repository. kaxilnaik pushed a commit to branch v3-1-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit caa07e2c12aee501639e44d2525542778973e865 Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Sep 14 16:20:58 2025 +0000 Separate GoLang precommit after prek supports monorepo setup (#54615) Prek 0.2.0 supports monorepo setup wieth separate pre-commit files and as the first step of separation we separate golang pre-commits. (cherry picked from commit d8e2c8559282508435647999fccd58c22c9108d9) --- .github/actions/install-prek/action.yml | 2 +- .github/workflows/ci-amd.yml | 2 +- .pre-commit-config.yaml | 45 +---- contributing-docs/08_static_code_checks.rst | 4 +- {.github => dev}/.pre-commit-config.yaml | 0 go-sdk/.pre-commit-config.yaml | 73 +++++++ scripts/ci/prek/upgrade_important_versions.py | 262 +++++++++++++++----------- 7 files changed, 235 insertions(+), 153 deletions(-) diff --git a/.github/actions/install-prek/action.yml b/.github/actions/install-prek/action.yml index cacd19f9639..9289c000aaa 100644 --- a/.github/actions/install-prek/action.yml +++ b/.github/actions/install-prek/action.yml @@ -64,7 +64,7 @@ runs: uses: apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468 with: # yamllint disable rule:line-length - key: cache-prek-v6-${{ inputs.platform }}-${{ inputs.python-version }}-${{inputs.skip-prek-hooks}}-${{ hashFiles('.pre-commit-config.yaml') }} + key: cache-prek-v6-${{ inputs.platform }}-${{ inputs.python-version }}-${{inputs.skip-prek-hooks}}-${{ hashFiles('**/.pre-commit-config.yaml') }} path: /tmp/ id: restore-prek-cache - name: "Check if prek cache tarball exists" diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index e65bc71e0fc..e7bd7010858 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -193,7 +193,7 @@ jobs: skip-prek-hooks: "" - name: "Run pin-versions" run: > - prek -c .github/.pre-commit-config.yaml --all-files --verbose --hook-stage manual + prek -c dev/.pre-commit-config.yaml --all-files --verbose --hook-stage manual pin-versions env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 93a75476896..4f008939a76 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,8 +19,7 @@ default_stages: [pre-commit, pre-push] default_language_version: python: python3 node: 22.19.0 - golang: 1.24.0 -minimum_prek_version: '0.1.3' +minimum_prek_version: '0.2.0' exclude: ^.*/.*_vendor/ repos: - repo: meta @@ -172,18 +171,6 @@ repos: - --fuzzy-match-generates-todo files: > \.cfg$|\.conf$|\.ini$|\.ldif$|\.properties$|\.service$|\.tf$|Dockerfile.*$ - - id: insert-license - name: Add license for all Go files - types: [go] - exclude: mocks/.*\.go$ - args: - - --comment-style - - "|//|" - - --license-filepath - - scripts/ci/license-templates/LICENSE.txt - - --insert-license-after-regex - # We need this 'generated by' line at the top for `golines` to not format it - - '// Code generated by .*' - repo: local hooks: - id: check-min-python-version @@ -1303,36 +1290,6 @@ repos: files: ^airflow-core/src/airflow/migrations/versions/.*\.py$ exclude: ^airflow-core/src/airflow/migrations/versions/0028_3_0_0_drop_ab_user_id_foreign_key.py$ - - id: go-mockery - name: Generate mocks for go - entry: -w /src/go-sdk vektra/mockery:3 - files: ^go-sdk/ - exclude: mocks/.*\.go$ - types: [go] - pass_filenames: false - language: docker_image - - id: go-mod-tidy - name: Run go mod tidy - entry: bash -c "cd go-sdk && go mod tidy" - files: ^go-sdk/ - exclude: mocks/.*\.go$ - pass_filenames: false - language: system - - id: gofmt - name: Format go code - entry: golines --base-formatter=gofumpt --write-output --max-len=100 --chain-split-dots - additional_dependencies: [github.com/segmentio/golines@latest, mvdan.cc/[email protected]] - files: ^go-sdk/ - types: [go] - language: golang - - id: gci - name: Consistent import ordering for Go files - # Since this is invoked from the root folder, not go-sdk/, gci can't auto-detect the prefix - entry: gci write --skip-generated -s standard -s default -s "prefix(github.com/apache/airflow)" - additional_dependencies: [github.com/daixiang0/[email protected]] - files: ^go-sdk/ - types: [go] - language: golang - id: ts-compile-lint-ui name: Compile / format / lint UI description: TS types generation / ESLint / Prettier new UI files diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst index 5e910a31a6b..297b2492830 100644 --- a/contributing-docs/08_static_code_checks.rst +++ b/contributing-docs/08_static_code_checks.rst @@ -275,14 +275,14 @@ GitHub Actions in the CI workflows. This action requires ``GITHUB_TOKEN`` to be set, otherwise you might hit the rate limits with GitHub API, it is also configured in a separate ``.prek-config.yaml`` file in the -``.github`` directory as it requires Python 3.11 to run. It is not run automatically +``dev`` directory as it requires Python 3.11 to run. It is not run automatically when you commit the code but in runs as a separate job in the CI. However, you can run it manually by running: .. code-block:: bash export GITHUB_TOKEN=YOUR_GITHUB_TOKEN - prek -c .github/.pre-commit-config.yaml --all-files --hook-stage manual --verbose + prek -c dev/.pre-commit-config.yaml --all-files --hook-stage manual --verbose Mypy checks diff --git a/.github/.pre-commit-config.yaml b/dev/.pre-commit-config.yaml similarity index 100% rename from .github/.pre-commit-config.yaml rename to dev/.pre-commit-config.yaml diff --git a/go-sdk/.pre-commit-config.yaml b/go-sdk/.pre-commit-config.yaml new file mode 100644 index 00000000000..4154c080168 --- /dev/null +++ b/go-sdk/.pre-commit-config.yaml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +default_language_version: + golang: 1.24.0 +minimum_prek_version: '0.0.28' +repos: + - repo: meta + hooks: + - id: identity + name: Print checked files + description: Print input to the static check hooks for troubleshooting + - id: check-hooks-apply + name: Check if all hooks apply to the repository + - repo: https://github.com/Lucas-C/pre-commit-hooks + # replace hash with version once PR #103 merged comes in a release + rev: abdd8b62891099da34162217ecb3872d22184a51 + hooks: + - id: insert-license + name: Add license for all Go files + types: [go] + exclude: mocks/.*\.go$ + args: + - --comment-style + - "|//|" + - --license-filepath + - ../scripts/ci/license-templates/LICENSE.txt + - --insert-license-after-regex + # We need this 'generated by' line at the top for `golines` to not format it + - '// Code generated by .*' + - repo: local + hooks: + - id: go-mockery + name: Generate mocks for go + entry: vektra/mockery:3 + exclude: mocks/.*\.go$ + types: [go] + pass_filenames: false + language: docker_image + - id: go-mod-tidy + name: Run go mod tidy + entry: go mod tidy + types: [go] + exclude: mocks/.*\.go$ + pass_filenames: false + language: golang + - id: gofmt + name: Format go code + entry: golines --base-formatter=gofumpt --write-output --max-len=100 --chain-split-dots + additional_dependencies: [github.com/segmentio/golines@latest, mvdan.cc/[email protected]] + types: [go] + language: golang + - id: gci + name: Consistent import ordering for Go files + entry: gci write --skip-generated -s standard -s default -s localmodule + additional_dependencies: [github.com/daixiang0/[email protected]] + types: [go] + language: golang diff --git a/scripts/ci/prek/upgrade_important_versions.py b/scripts/ci/prek/upgrade_important_versions.py index 75dc99e1be2..779c692b70b 100755 --- a/scripts/ci/prek/upgrade_important_versions.py +++ b/scripts/ci/prek/upgrade_important_versions.py @@ -24,6 +24,13 @@ # "rich>=13.6.0", # ] # /// +# +# DEBUGGING +# * You can set UPGRADE_ALL_BY_DEFAULT to "false" to only upgrade those versions that +# are set by UPGRADE_NNNNNNN (NNNNNN > thing to upgrade version) +# * You can set VERBOSE="true" to see requests being made +# * You can set UPGRADE_NNNNNNN_INCLUDE_PRE_RELEASES="true" + from __future__ import annotations import os @@ -79,46 +86,82 @@ for file in DOCKER_IMAGES_EXAMPLE_DIR_PATH.rglob("*.sh"): FILES_TO_UPDATE.append((file, False)) -def get_latest_pypi_version(package_name: str) -> str: +def get_latest_pypi_version(package_name: str, should_upgrade: bool) -> str: + if not should_upgrade: + return "" + if VERBOSE: + console.print(f"[bright_blue]Fetching latest version for {package_name} from PyPI") response = requests.get( f"https://pypi.org/pypi/{package_name}/json";, headers={"User-Agent": "Python requests"} ) response.raise_for_status() # Ensure we got a successful response data = response.json() - latest_version = data["info"]["version"] # The version info is under the 'info' key + if os.environ.get(f"UPGRADE_{package_name.upper()}_INCLUDE_PRE_RELEASES", ""): + latest_version = str(sorted([Version(version) for version in data["releases"].keys()])[-1]) + else: + latest_version = data["info"]["version"] # The version info is under the 'info' key + if VERBOSE: + console.print(f"[bright_blue]Latest version for {package_name}: {latest_version}") return latest_version -def get_latest_python_version(python_major_minor: str, github_token: str | None) -> str | None: - latest_version = None - # Matches versions of vA.B.C and vA.B where C can only be numeric and v is optional - version_match = re.compile(rf"^v?{python_major_minor}\.?\d*$") +def get_all_python_versions() -> list[Version]: + if VERBOSE: + console.print("[bright_blue]Fetching all released Python versions from python.org") + url = "https://www.python.org/api/v2/downloads/release/?is_published=true"; headers = {"User-Agent": "Python requests"} - if github_token: - headers["Authorization"] = f"Bearer {github_token}" - for i in range(5): - response = requests.get( - f"https://api.github.com/repos/python/cpython/tags?per_page=100&page={i + 1}", - headers=headers, - ) - response.raise_for_status() # Ensure we got a successful response - data = response.json() - versions = [str(tag["name"]) for tag in data if version_match.match(tag.get("name", ""))] - if versions: - latest_version = sorted(versions, key=Version, reverse=True)[0] - break - return latest_version[1:] if latest_version and latest_version.startswith("v") else latest_version + response = requests.get(url, headers=headers) + response.raise_for_status() + data = response.json() + versions = [] + matcher = re.compile(r"^Python ([\d.]+$)") + for release in data: + release_name = release["name"] + match = matcher.match(release_name) + if match: + versions.append(Version(match.group(1))) + return versions + + +def get_latest_python_version(python_major_minor: str, all_versions: list[Version]) -> str: + """ + Fetch the latest released Python version for a given major.minor (e.g. '3.12') using python.org API. + Much faster than paginating through all GitHub tags. + """ + # Only consider releases matching the major.minor.patch pattern + matching = [ + version for version in all_versions if python_major_minor == f"{version.major}.{version.minor}" + ] + if not matching: + console.print(f"[bright_red]No released Python versions found for {python_major_minor}") + sys.exit(1) + # Sort and return the latest version + latest_version = sorted(matching)[-1] + if VERBOSE: + console.print(f"[bright_blue]Latest version for {python_major_minor}: {latest_version}") + return str(latest_version) def get_latest_golang_version() -> str: + if not UPGRADE_GOLANG: + return "" + if VERBOSE: + console.print("[bright_blue]Fetching latest Go version from go.dev") response = requests.get("https://go.dev/dl/?mode=json";) response.raise_for_status() # Ensure we got a successful response versions = response.json() stable_versions = [release["version"].replace("go", "") for release in versions if release["stable"]] - return sorted(stable_versions, key=Version, reverse=True)[0] + latest_version = sorted(stable_versions, key=Version, reverse=True)[0] + if VERBOSE: + console.print(f"[bright_blue]Latest version for Go: {latest_version}") + return latest_version def get_latest_lts_node_version() -> str: + if not UPGRADE_NODE_LTS: + return "" + if VERBOSE: + console.print("[bright_blue]Fetching latest LTS Node version from nodejs.org") response = requests.get("https://nodejs.org/dist/index.json";) response.raise_for_status() # Ensure we got a successful response versions = response.json() @@ -126,7 +169,10 @@ def get_latest_lts_node_version() -> str: lts_versions = [version["version"] for version in versions if version["version"].startswith(lts_prefix)] # The json array is sorted from newest to oldest, so the first element is the latest LTS version # Skip leading v in version - return lts_versions[0][1:] + latest_version = lts_versions[0][1:] + if VERBOSE: + console.print(f"[bright_blue]Latest version for LTS Node: {latest_version}") + return latest_version class Quoting(Enum): @@ -138,38 +184,44 @@ class Quoting(Enum): PIP_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ - (re.compile(r"(AIRFLOW_PIP_VERSION=)([0-9.]+)"), Quoting.UNQUOTED), - (re.compile(r"(python -m pip install --upgrade pip==)([0-9.]+)"), Quoting.UNQUOTED), - (re.compile(r"(AIRFLOW_PIP_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), - (re.compile(r"(PIP_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), - (re.compile(r"(PIP_VERSION=)(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), - (re.compile(r"(\| *`AIRFLOW_PIP_VERSION` *\| *)(`[0-9.]+`)( *\|)"), Quoting.REVERSE_SINGLE_QUOTED), + (re.compile(r"(AIRFLOW_PIP_VERSION=)([0-9.abrc]+)"), Quoting.UNQUOTED), + (re.compile(r"(python -m pip install --upgrade pip==)([0-9.abrc]+)"), Quoting.UNQUOTED), + (re.compile(r"(AIRFLOW_PIP_VERSION = )(\"[0-9.abrc]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(PIP_VERSION = )(\"[0-9.abrc]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(PIP_VERSION=)(\"[0-9.abrc]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(\| *`AIRFLOW_PIP_VERSION` *\| *)(`[0-9.abrc]+`)( *\|)"), Quoting.REVERSE_SINGLE_QUOTED), ] PYTHON_PATTERNS: list[tuple[str, Quoting]] = [ - (r"(\"{python_major_minor}\": \")([0-9.]+)(\")", Quoting.UNQUOTED), + (r"(\"{python_major_minor}\": \")([0-9.abrc]+)(\")", Quoting.UNQUOTED), ] GOLANG_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ - (re.compile(r"(GOLANG_MAJOR_MINOR_VERSION=)(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), - (re.compile(r"(\| *`GOLANG_MAJOR_MINOR_VERSION` *\| *)(`[0-9.]+`)( *\|)"), Quoting.REVERSE_SINGLE_QUOTED), + (re.compile(r"(GOLANG_MAJOR_MINOR_VERSION=)(\"[0-9.abrc]+\")"), Quoting.DOUBLE_QUOTED), + ( + re.compile(r"(\| *`GOLANG_MAJOR_MINOR_VERSION` *\| *)(`[0-9.abrc]+`)( *\|)"), + Quoting.REVERSE_SINGLE_QUOTED, + ), ] AIRFLOW_IMAGE_PYTHON_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ - (re.compile(r"(AIRFLOW_PYTHON_VERSION=)(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), - (re.compile(r"(\| ``AIRFLOW_PYTHON_VERSION`` *\| )(``[0-9.]+``)( *\|)"), Quoting.REVERSE_DOUBLE_QUOTED), + (re.compile(r"(AIRFLOW_PYTHON_VERSION=)(\"[0-9.abrc]+\")"), Quoting.DOUBLE_QUOTED), + ( + re.compile(r"(\| ``AIRFLOW_PYTHON_VERSION`` *\| )(``[0-9.abrc]+``)( *\|)"), + Quoting.REVERSE_DOUBLE_QUOTED, + ), ] UV_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ - (re.compile(r"(AIRFLOW_UV_VERSION=)([0-9.]+)"), Quoting.UNQUOTED), - (re.compile(r"(uv>=)([0-9.]+)"), Quoting.UNQUOTED), - (re.compile(r"(AIRFLOW_UV_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), - (re.compile(r"^(\s*UV_VERSION = )(\"[0-9.]+\")", re.MULTILINE), Quoting.DOUBLE_QUOTED), - (re.compile(r"^(\s*UV_VERSION=)(\"[0-9.]+\")", re.MULTILINE), Quoting.DOUBLE_QUOTED), - (re.compile(r"(\| *`AIRFLOW_UV_VERSION` *\| *)(`[0-9.]+`)( *\|)"), Quoting.REVERSE_SINGLE_QUOTED), + (re.compile(r"(AIRFLOW_UV_VERSION=)([0-9.abrc]+)"), Quoting.UNQUOTED), + (re.compile(r"(uv>=)([0-9.abrc]+)"), Quoting.UNQUOTED), + (re.compile(r"(AIRFLOW_UV_VERSION = )(\"[0-9.abrc]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"^(\s*UV_VERSION = )(\"[0-9.abrc]+\")", re.MULTILINE), Quoting.DOUBLE_QUOTED), + (re.compile(r"^(\s*UV_VERSION=)(\"[0-9.abrc]+\")", re.MULTILINE), Quoting.DOUBLE_QUOTED), + (re.compile(r"(\| *`AIRFLOW_UV_VERSION` *\| *)(`[0-9.abrd]+`)( *\|)"), Quoting.REVERSE_SINGLE_QUOTED), ( re.compile( - r"(\")([0-9.]+)(\" # Keep this comment to " + r"(\")([0-9.abrc]+)(\" # Keep this comment to " r"allow automatic replacement of uv version)" ), Quoting.UNQUOTED, @@ -177,18 +229,18 @@ UV_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ ] PREK_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ - (re.compile(r"(AIRFLOW_PREK_VERSION=)([0-9.]+)"), Quoting.UNQUOTED), - (re.compile(r"(AIRFLOW_PREK_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), - (re.compile(r"(prek>=)([0-9.]+)"), Quoting.UNQUOTED), - (re.compile(r"(PREK_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), - (re.compile(r"(PREK_VERSION=)(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(AIRFLOW_PREK_VERSION=)([0-9.abrc]+)"), Quoting.UNQUOTED), + (re.compile(r"(AIRFLOW_PREK_VERSION = )(\"[0-9.abrc]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(prek>=)([0-9.abrc]+)"), Quoting.UNQUOTED), + (re.compile(r"(PREK_VERSION = )(\"[0-9.abrc]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(PREK_VERSION=)(\"[0-9.abrc]+\")"), Quoting.DOUBLE_QUOTED), ( - re.compile(r"(\| *`AIRFLOW_PREK_VERSION` *\| *)(`[0-9.]+`)( *\|)"), + re.compile(r"(\| *`AIRFLOW_PREK_VERSION` *\| *)(`[0-9.abrc]+`)( *\|)"), Quoting.REVERSE_SINGLE_QUOTED, ), ( re.compile( - r"(\")([0-9.]+)(\" # Keep this comment to allow automatic " + r"(\")([0-9.abrc]+)(\" # Keep this comment to allow automatic " r"replacement of prek version)" ), Quoting.UNQUOTED, @@ -196,7 +248,7 @@ PREK_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ ] NODE_LTS_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ - (re.compile(r"( *node: )([0-9.]+)"), Quoting.UNQUOTED), + (re.compile(r"(^ node: )([0-9.abrc]+)^"), Quoting.UNQUOTED), ] @@ -212,18 +264,25 @@ def get_replacement(value: str, quoting: Quoting) -> str: return value -UPGRADE_UV: bool = os.environ.get("UPGRADE_UV", "true").lower() == "true" -UPGRADE_PIP: bool = os.environ.get("UPGRADE_PIP", "true").lower() == "true" -UPGRADE_PYTHON: bool = os.environ.get("UPGRADE_PYTHON", "true").lower() == "true" -UPGRADE_GOLANG: bool = os.environ.get("UPGRADE_GOLANG", "true").lower() == "true" -UPGRADE_PREK: bool = os.environ.get("UPGRADE_PREK", "true").lower() == "true" -UPGRADE_NODE_LTS: bool = os.environ.get("UPGRADE_NODE_LTS", "true").lower() == "true" -UPGRADE_HATCH: bool = os.environ.get("UPGRADE_HATCH", "true").lower() == "true" -UPGRADE_PYYAML: bool = os.environ.get("UPGRADE_PYYAML", "true").lower() == "true" -UPGRADE_GITPYTHON: bool = os.environ.get("UPGRADE_GITPYTHON", "true").lower() == "true" -UPGRADE_RICH: bool = os.environ.get("UPGRADE_RICH", "true").lower() == "true" -UPGRADE_RUFF: bool = os.environ.get("UPGRADE_RUFF", "true").lower() == "true" -UPGRADE_MYPY: bool = os.environ.get("UPGRADE_MYPY", "true").lower() == "true" +VERBOSE: bool = os.environ.get("VERBOSE", "false") == "true" +UPGRADE_ALL_BY_DEFAULT: bool = os.environ.get("UPGRADE_ALL_BY_DEFAULT", "true") == "true" +UPGRADE_ALL_BY_DEFAULT_STR: str = str(UPGRADE_ALL_BY_DEFAULT).lower() +if UPGRADE_ALL_BY_DEFAULT: + if VERBOSE == "true": + console.print("[bright_blue]Upgrading all important versions") + +UPGRADE_GITPYTHON: bool = os.environ.get("UPGRADE_GITPYTHON", UPGRADE_ALL_BY_DEFAULT_STR).lower() == "true" +UPGRADE_GOLANG: bool = os.environ.get("UPGRADE_GOLANG", UPGRADE_ALL_BY_DEFAULT_STR).lower() == "true" +UPGRADE_HATCH: bool = os.environ.get("UPGRADE_HATCH", UPGRADE_ALL_BY_DEFAULT_STR).lower() == "true" +UPGRADE_NODE_LTS: bool = os.environ.get("UPGRADE_NODE_LTS", UPGRADE_ALL_BY_DEFAULT_STR).lower() == "true" +UPGRADE_PIP: bool = os.environ.get("UPGRADE_PIP", UPGRADE_ALL_BY_DEFAULT_STR).lower() == "true" +UPGRADE_PREK: bool = os.environ.get("UPGRADE_PREK", UPGRADE_ALL_BY_DEFAULT_STR).lower() == "true" +UPGRADE_PYTHON: bool = os.environ.get("UPGRADE_PYTHON", UPGRADE_ALL_BY_DEFAULT_STR).lower() == "true" +UPGRADE_PYYAML: bool = os.environ.get("UPGRADE_PYYAML", UPGRADE_ALL_BY_DEFAULT_STR).lower() == "true" +UPGRADE_RICH: bool = os.environ.get("UPGRADE_RICH", UPGRADE_ALL_BY_DEFAULT_STR).lower() == "true" +UPGRADE_RUFF: bool = os.environ.get("UPGRADE_RUFF", UPGRADE_ALL_BY_DEFAULT_STR).lower() == "true" +UPGRADE_UV: bool = os.environ.get("UPGRADE_UV", UPGRADE_ALL_BY_DEFAULT_STR).lower() == "true" +UPGRADE_MYPY: bool = os.environ.get("UPGRADE_MYPY", UPGRADE_ALL_BY_DEFAULT_STR).lower() == "true" ALL_PYTHON_MAJOR_MINOR_VERSIONS = ["3.9", "3.10", "3.11", "3.12", "3.13"] DEFAULT_PROD_IMAGE_PYTHON_VERSION = "3.12" @@ -260,51 +319,53 @@ if __name__ == "__main__": gh_token = retrieve_gh_token(description="airflow-upgrade-important-versions", scopes="public_repo") changed = False golang_version = get_latest_golang_version() - pip_version = get_latest_pypi_version("pip") - uv_version = get_latest_pypi_version("uv") - setuptools_version = get_latest_pypi_version("setuptools") - prek_version = get_latest_pypi_version("prek") - hatch_version = get_latest_pypi_version("hatch") - pyyaml_version = get_latest_pypi_version("PyYAML") - gitpython_version = get_latest_pypi_version("GitPython") - rich_version = get_latest_pypi_version("rich") - ruff_version = get_latest_pypi_version("ruff") + pip_version = get_latest_pypi_version("pip", UPGRADE_PIP) + uv_version = get_latest_pypi_version("uv", UPGRADE_UV) + prek_version = get_latest_pypi_version("prek", UPGRADE_PREK) + hatch_version = get_latest_pypi_version("hatch", UPGRADE_HATCH) + pyyaml_version = get_latest_pypi_version("PyYAML", UPGRADE_PYYAML) + gitpython_version = get_latest_pypi_version("GitPython", UPGRADE_GITPYTHON) + ruff_version = get_latest_pypi_version("ruff", UPGRADE_RUFF) + rich_version = get_latest_pypi_version("rich", UPGRADE_RICH) + mypy_version = get_latest_pypi_version("mypy", UPGRADE_MYPY) node_lts_version = get_latest_lts_node_version() - mypy_version = get_latest_pypi_version("mypy") + latest_python_versions: dict[str, str] = {} + latest_image_python_version = "" + if UPGRADE_PYTHON: + all_python_versions = get_all_python_versions() if UPGRADE_PYTHON else None + for python_major_minor_version in ALL_PYTHON_MAJOR_MINOR_VERSIONS: + latest_python_versions[python_major_minor_version] = get_latest_python_version( + python_major_minor_version, all_python_versions + ) + if python_major_minor_version == DEFAULT_PROD_IMAGE_PYTHON_VERSION: + latest_image_python_version = latest_python_versions[python_major_minor_version] + console.print( + f"[bright_blue]Latest image python {python_major_minor_version} version: {latest_image_python_version}" + ) for file, keep_length in FILES_TO_UPDATE: console.print(f"[bright_blue]Updating {file}") file_content = file.read_text() new_content = file_content if UPGRADE_PIP: - console.print(f"[bright_blue]Latest pip version: {pip_version}") for line_pattern, quoting in PIP_PATTERNS: new_content = replace_version( line_pattern, get_replacement(pip_version, quoting), new_content, keep_length ) if UPGRADE_PYTHON: - for python_version in ALL_PYTHON_MAJOR_MINOR_VERSIONS: - latest_python_version = get_latest_python_version(python_version, gh_token) - if not latest_python_version: - console.print( - f"[red]Failed to get latest version for python {python_version}. Skipping.[/]" - ) - sys.exit(1) - console.print(f"[bright_blue]Latest python {python_version} version: {latest_python_version}") + for python_major_minor_version in ALL_PYTHON_MAJOR_MINOR_VERSIONS: + latest_python_version = latest_python_versions[python_major_minor_version] for line_format, quoting in PYTHON_PATTERNS: - line_pattern = re.compile(line_format.format(python_major_minor=python_version)) - console.print(line_pattern) + line_pattern = re.compile( + line_format.format(python_major_minor=python_major_minor_version) + ) new_content = replace_version( line_pattern, get_replacement(latest_python_version, quoting), new_content, keep_length, ) - if python_version == DEFAULT_PROD_IMAGE_PYTHON_VERSION: - console.print( - f"[bright_blue]Latest image python {python_version} version: {latest_python_version}" - ) + if python_major_minor_version == DEFAULT_PROD_IMAGE_PYTHON_VERSION: for line_pattern, quoting in AIRFLOW_IMAGE_PYTHON_PATTERNS: - console.print(line_pattern) new_content = replace_version( line_pattern, get_replacement(latest_python_version, quoting), @@ -312,25 +373,21 @@ if __name__ == "__main__": keep_length, ) if UPGRADE_GOLANG: - console.print(f"[bright_blue]Latest golang version: {golang_version}") for line_pattern, quoting in GOLANG_PATTERNS: new_content = replace_version( line_pattern, get_replacement(golang_version, quoting), new_content, keep_length ) if UPGRADE_UV: - console.print(f"[bright_blue]Latest uv version: {uv_version}") for line_pattern, quoting in UV_PATTERNS: new_content = replace_version( line_pattern, get_replacement(uv_version, quoting), new_content, keep_length ) if UPGRADE_PREK: - console.print(f"[bright_blue]Latest prek version: {prek_version}") for line_pattern, quoting in PREK_PATTERNS: new_content = replace_version( line_pattern, get_replacement(prek_version, quoting), new_content, keep_length ) if UPGRADE_NODE_LTS: - console.print(f"[bright_blue]Latest Node LTS version: {node_lts_version}") for line_pattern, quoting in NODE_LTS_PATTERNS: new_content = replace_version( line_pattern, @@ -339,62 +396,57 @@ if __name__ == "__main__": keep_length, ) if UPGRADE_HATCH: - console.print(f"[bright_blue]Latest hatch version: {hatch_version}") new_content = re.sub( - r"(HATCH_VERSION = )(\"[0-9.]+\")", + r"(HATCH_VERSION = )(\"[0-9.abrc]+\")", f'HATCH_VERSION = "{hatch_version}"', new_content, ) new_content = re.sub( - r"(HATCH_VERSION=)(\"[0-9.]+\")", + r"(HATCH_VERSION=)(\"[0-9.abrc]+\")", f'HATCH_VERSION="{hatch_version}"', new_content, ) if UPGRADE_PYYAML: - console.print(f"[bright_blue]Latest PyYAML version: {pyyaml_version}") new_content = re.sub( - r"(PYYAML_VERSION = )(\"[0-9.]+\")", + r"(PYYAML_VERSION = )(\"[0-9.abrc]+\")", f'PYYAML_VERSION = "{pyyaml_version}"', new_content, ) new_content = re.sub( - r"(PYYAML_VERSION=)(\"[0-9.]+\")", + r"(PYYAML_VERSION=)(\"[0-9.abrc]+\")", f'PYYAML_VERSION="{pyyaml_version}"', new_content, ) if UPGRADE_GITPYTHON: - console.print(f"[bright_blue]Latest GitPython version: {gitpython_version}") new_content = re.sub( - r"(GITPYTHON_VERSION = )(\"[0-9.]+\")", + r"(GITPYTHON_VERSION = )(\"[0-9.abrc]+\")", f'GITPYTHON_VERSION = "{gitpython_version}"', new_content, ) new_content = re.sub( - r"(GITPYTHON_VERSION=)(\"[0-9.]+\")", + r"(GITPYTHON_VERSION=)(\"[0-9.abrc]+\")", f'GITPYTHON_VERSION="{gitpython_version}"', new_content, ) if UPGRADE_RICH: - console.print(f"[bright_blue]Latest rich version: {rich_version}") new_content = re.sub( - r"(RICH_VERSION = )(\"[0-9.]+\")", + r"(RICH_VERSION = )(\"[0-9.abrc]+\")", f'RICH_VERSION = "{rich_version}"', new_content, ) new_content = re.sub( - r"(RICH_VERSION=)(\"[0-9.]+\")", + r"(RICH_VERSION=)(\"[0-9.abrc]+\")", f'RICH_VERSION="{rich_version}"', new_content, ) if UPGRADE_RUFF: - console.print(f"[bright_blue]Latest ruff version: {ruff_version}") new_content = re.sub( - r"(ruff==)([0-9.]+)", + r"(ruff==)([0-9.abrc]+)", f"ruff=={ruff_version}", new_content, ) new_content = re.sub( - r"(ruff>=)([0-9.]+)", + r"(ruff>=)([0-9.abrc]+)", f"ruff>={ruff_version}", new_content, )
