This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push: new ca7299e88c Fix few caveats in provider rlease process (#36264) ca7299e88c is described below commit ca7299e88ca6a33d62ae806e0e9263f71e34ceb3 Author: Jarek Potiuk <ja...@potiuk.com> AuthorDate: Sun Dec 17 19:24:16 2023 +0100 Fix few caveats in provider rlease process (#36264) There were few small problems in release process: 1) When the SVN directories were in symbolically linked directories, svn got into locked directory status 2) Cleaning old relases failed to find the files in current directory, because breeze changes working directory before running and the `.` directory was really `airflow` repository, thus no released artifacts could be found. 3) When the old releases contained `_` rather thatn `-` (as it was in the bdist_wheel prepared packages) then we failed to delet the files because the file names for them were constructed from the normalized file names (with `-`). This PR fixes those problems: 1) `VAR=$(pwd -P)` is used in order to resolve potentially symbolically linked urrent working directory and `cd ${VAR}` is used to switch dir to them 2) The same `$(pwd -P)` is used to pass the directory to the clean-old-provider-artifacts command to avoid interpreting `.` as "AIRFLOW_ROOT" directory. 3) Filen name is stored in FileVersion and used to run actual `svn rm` command - this way the right file name will be used, regardless from normalization of the package names. --- dev/README_RELEASE_PROVIDER_PACKAGES.md | 16 +++++++++------- .../commands/release_management_commands.py | 8 +++++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index 16d6fbe9f9..0a437285cf 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -248,7 +248,7 @@ generates corresponding .asc and .sha512 files for each file to sign. * Cleanup dist folder: ```shell script -export AIRFLOW_REPO_ROOT=$(pwd) +export AIRFLOW_REPO_ROOT=$(pwd -P) rm -rf ${AIRFLOW_REPO_ROOT}/dist/* ``` @@ -401,7 +401,7 @@ lists and should be updated every time a new version of provider packages is rel ```shell script git clone https://github.com/apache/airflow-site.git airflow-site cd airflow-site -export AIRFLOW_SITE_DIRECTORY="$(pwd)" +export AIRFLOW_SITE_DIRECTORY="$(pwd -P)" ``` Note if this is not the first time you clone the repo make sure main branch is rebased: @@ -961,14 +961,16 @@ We also need to archive older releases before copying the new ones ```bash cd "<ROOT_OF_YOUR_AIRFLOW_REPO>" # Set AIRFLOW_REPO_ROOT to the path of your git repo -export AIRFLOW_REPO_ROOT="$(pwd)" +export AIRFLOW_REPO_ROOT="$(pwd -P)" # Go the folder where you have checked out the release repo from SVN # Make sure this is direct directory and a symbolic link # Otherwise 'svn mv' errors out if it is with "E200033: Another process is blocking the working copy database cd "<ROOT_WHERE_YOUR_ASF_DIST_IS_CREATED>" -export ASF_DIST_PARENT="$(pwd)" +export ASF_DIST_PARENT="$(pwd -P)" +# make sure physical path is used, in case original directory is symbolically linked +cd "${ASF_DIST_PARENT}" # or clone it if it's not done yet [ -d asf-dist ] || svn checkout --depth=immediates https://dist.apache.org/repos/dist asf-dist @@ -985,7 +987,7 @@ svn rm ${SOURCE_DIR}/*<provider>* # Create providers folder if it does not exist # All latest releases are kept in this one folder without version sub-folder -cd asf-dist/release/airflow +cd "${ASF_DIST_PARENT}/asf-dist/release/airflow" mkdir -pv providers cd providers @@ -1000,10 +1002,10 @@ do done # Check which old packages will be removed using dry run -breeze release-management clean-old-provider-artifacts --directory . --dry-run +breeze release-management clean-old-provider-artifacts --directory $(pwd -P) --dry-run # Remove those packages -breeze release-management clean-old-provider-artifacts --directory . +breeze release-management clean-old-provider-artifacts --directory $(pwd -P) # You need to do go to the asf-dist directory in order to commit both dev and release together cd ${ASF_DIST_PARENT}/asf-dist diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index f527ee4b27..3af12990f2 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -202,6 +202,7 @@ class VersionedFile(NamedTuple): suffix: str type: str comparable_version: Version + file_name: str AIRFLOW_PIP_VERSION = "23.3.1" @@ -1291,10 +1292,10 @@ def clean_old_provider_artifacts( # Leave only last version from each type for versioned_file in package_types[:-1]: get_console().print( - f"""[warning]Removing {versioned_file.base + versioned_file.version + - versioned_file.suffix} as they are older than remaining file""" + f"[warning]Removing {versioned_file.file_name} as they are older than remaining file: " + f"{package_types[-1].file_name}[/]" ) - command = ["svn", "rm", versioned_file.base + versioned_file.version + versioned_file.suffix] + command = ["svn", "rm", versioned_file.file_name] run_command(command, check=False) @@ -1939,4 +1940,5 @@ def split_version_and_suffix(file_name: str, suffix: str) -> VersionedFile: suffix=suffix, type=no_version_file + "-" + suffix, comparable_version=Version(version), + file_name=file_name, )