This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 0709cd7e0b GH-45920: [Release][Python] Upload sdist and wheels to
GitHub Releases not apache.jfrog.io (#45962)
0709cd7e0b is described below
commit 0709cd7e0b866db25f13554b8ded7ef3693c55e3
Author: Sutou Kouhei <[email protected]>
AuthorDate: Sun Mar 30 10:35:06 2025 +0900
GH-45920: [Release][Python] Upload sdist and wheels to GitHub Releases not
apache.jfrog.io (#45962)
### Rationale for this change
We want to stop using apache.jfrog.io. See also: #40760
### What changes are included in this PR?
Use GitHub Release instead of apache.jfrog.io.
Users use only sdist and wheels published to PyPI. Publishing to PyPI isn't
changed. So we don't need any migration path for users.
### Are these changes tested?
No. I want to try this in the next release.
### Are there any user-facing changes?
No.
* GitHub Issue: #45920
Authored-by: Sutou Kouhei <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
.pre-commit-config.yaml | 18 +++++
dev/release/.env.example => .shellcheckrc | 29 +-------
dev/release/.env.example | 5 ++
dev/release/05-binary-upload.sh | 106 +++++++++++++++++++-----------
dev/release/binary-task.rb | 9 ---
dev/release/post-03-binary.sh | 61 +++++++++--------
dev/release/post-11-python.sh | 38 ++++++-----
7 files changed, 146 insertions(+), 120 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 33fc8b8717..1b6cd35287 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -195,9 +195,27 @@ repos:
?^ci/scripts/c_glib_build\.sh$|
?^ci/scripts/c_glib_test\.sh$|
?^c_glib/test/run-test\.sh$|
+ ?^dev/release/05-binary-upload\.sh$|
?^dev/release/07-binary-verify\.sh$|
+ ?^dev/release/post-03-binary\.sh$|
+ ?^dev/release/post-11-python\.sh$|
?^dev/release/utils-generate-checksum\.sh$|
)
+ - repo: https://github.com/scop/pre-commit-shfmt
+ rev: v3.11.0-1
+ hooks:
+ - id: shfmt
+ args:
+ # The default args is "--write --simplify" but we don't use
+ # "--simplify". Because it's conflicted will ShellCheck.
+ - "--write"
+ # TODO: Remove this when we fix all lint failures
+ files: >-
+ (
+ ?^dev/release/05-binary-upload\.sh$|
+ ?^dev/release/post-03-binary\.sh$|
+ ?^dev/release/post-11-python\.sh$|
+ )
- repo: https://github.com/trim21/pre-commit-mirror-meson
rev: v1.6.1
hooks:
diff --git a/dev/release/.env.example b/.shellcheckrc
similarity index 51%
copy from dev/release/.env.example
copy to .shellcheckrc
index 116a39eb56..6eb7b22b94 100644
--- a/dev/release/.env.example
+++ b/.shellcheckrc
@@ -15,30 +15,5 @@
# specific language governing permissions and limitations
# under the License.
-# The GPG key ID to sign artifacts. The GPG key ID must be registered
-# to both of the followings:
-#
-# * https://dist.apache.org/repos/dist/dev/arrow/KEYS
-# * https://dist.apache.org/repos/dist/release/arrow/KEYS
-#
-# See these files how to import your GPG key ID to these files.
-#
-# You must set this.
-#GPG_KEY_ID=08D3564B7C6A9CAFBFF6A66791D18FCF079F8007
-
-# The Artifactory API key to upload artifacts to Artifactory.
-#
-# You must set this.
-#ARTIFACTORY_API_KEY=secret
-
-# The Apache Sofotware Foundation ID to upload artifacts to
-# repository.apache.org.
-#
-# You must set this.
-#ASF_USER=kou
-
-# The Apache Sofotware Foundation password to upload artifacts to
-# repository.apache.org.
-#
-# You must set this.
-#ASF_PASSWORD=secret
+external-sources=true
+source-path=SCRIPTDIR
diff --git a/dev/release/.env.example b/dev/release/.env.example
index 116a39eb56..e751ce4dbc 100644
--- a/dev/release/.env.example
+++ b/dev/release/.env.example
@@ -42,3 +42,8 @@
#
# You must set this.
#ASF_PASSWORD=secret
+
+# The GitHub token to upload artifacts to GitHub Release.
+#
+# You must set this.
+#GH_TOKEN=secret
diff --git a/dev/release/05-binary-upload.sh b/dev/release/05-binary-upload.sh
index 1e768c6e35..3c27242862 100755
--- a/dev/release/05-binary-upload.sh
+++ b/dev/release/05-binary-upload.sh
@@ -38,9 +38,10 @@ version_with_rc="${version}-rc${rc}"
crossbow_job_prefix="release-${version_with_rc}"
crossbow_package_dir="${SOURCE_DIR}/../../packages"
-: ${CROSSBOW_JOB_NUMBER:="0"}
-: ${CROSSBOW_JOB_ID:="${crossbow_job_prefix}-${CROSSBOW_JOB_NUMBER}"}
-: ${ARROW_ARTIFACTS_DIR:="${crossbow_package_dir}/${CROSSBOW_JOB_ID}"}
+: "${CROSSBOW_JOB_NUMBER:=0}"
+: "${CROSSBOW_JOB_ID:=${crossbow_job_prefix}-${CROSSBOW_JOB_NUMBER}}"
+: "${ARROW_ARTIFACTS_DIR:=${crossbow_package_dir}/${CROSSBOW_JOB_ID}}"
+: "${GITHUB_REPOSITORY:=apache/arrow}"
if [ ! -e "${ARROW_ARTIFACTS_DIR}" ]; then
echo "${ARROW_ARTIFACTS_DIR} does not exist"
@@ -59,6 +60,7 @@ if [ ! -f .env ]; then
echo "You can use $(pwd)/.env.example as template"
exit 1
fi
+# shellcheck source=SCRIPTDIR/.env.example
. .env
. utils-binary.sh
@@ -66,52 +68,74 @@ fi
# By default upload all artifacts.
# To deactivate one category, deactivate the category and all of its
dependents.
# To explicitly select one category, set UPLOAD_DEFAULT=0 UPLOAD_X=1.
-: ${UPLOAD_DEFAULT:=1}
-: ${UPLOAD_ALMALINUX:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_AMAZON_LINUX:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_CENTOS:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_DEBIAN:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_DOCS:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_PYTHON:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_R:=${UPLOAD_DEFAULT}}
-: ${UPLOAD_UBUNTU:=${UPLOAD_DEFAULT}}
+: "${UPLOAD_DEFAULT:=1}"
+: "${UPLOAD_ALMALINUX:=${UPLOAD_DEFAULT}}"
+: "${UPLOAD_AMAZON_LINUX:=${UPLOAD_DEFAULT}}"
+: "${UPLOAD_CENTOS:=${UPLOAD_DEFAULT}}"
+: "${UPLOAD_DEBIAN:=${UPLOAD_DEFAULT}}"
+: "${UPLOAD_DOCS:=${UPLOAD_DEFAULT}}"
+: "${UPLOAD_PYTHON:=${UPLOAD_DEFAULT}}"
+: "${UPLOAD_R:=${UPLOAD_DEFAULT}}"
+: "${UPLOAD_UBUNTU:=${UPLOAD_DEFAULT}}"
+
+tmp_dir=binary/tmp
+rm -rf "${tmp_dir}"
+mkdir -p "${tmp_dir}"
+
+if [ "${UPLOAD_PYTHON}" -gt 0 ]; then
+ dist_dir="${tmp_dir}/dist"
+ mkdir -p "${dist_dir}"
+ for target in "${ARROW_ARTIFACTS_DIR}"/python-sdist/* \
+ "${ARROW_ARTIFACTS_DIR}"/wheel-*/*; do
+ base_name="$(basename "${target}")"
+ cp -a "${target}" "${dist_dir}/${base_name}"
+ gpg \
+ --armor \
+ --detach-sign \
+ --local-user "${GPG_KEY_ID}" \
+ --output "${dist_dir}/${base_name}.asc" \
+ "${target}"
+ pushd "${dist_dir}"
+ shasum -a 512 "${base_name}" >"${base_name}.sha512"
+ popd
+ done
+ gh release upload \
+ --repo apache/arrow \
+ "apache-arrow-${version}-rc${rc}" \
+ "${dist_dir}"/*
+fi
rake_tasks=()
apt_targets=()
yum_targets=()
-if [ ${UPLOAD_ALMALINUX} -gt 0 ]; then
+if [ "${UPLOAD_ALMALINUX}" -gt 0 ]; then
rake_tasks+=(yum:rc:artifactory yum:rc)
yum_targets+=(almalinux)
fi
-if [ ${UPLOAD_AMAZON_LINUX} -gt 0 ]; then
+if [ "${UPLOAD_AMAZON_LINUX}" -gt 0 ]; then
rake_tasks+=(yum:rc:artifactory yum:rc)
yum_targets+=(amazon-linux)
fi
-if [ ${UPLOAD_CENTOS} -gt 0 ]; then
+if [ "${UPLOAD_CENTOS}" -gt 0 ]; then
rake_tasks+=(yum:rc:artifactory yum:rc)
yum_targets+=(centos)
fi
-if [ ${UPLOAD_DEBIAN} -gt 0 ]; then
+if [ "${UPLOAD_DEBIAN}" -gt 0 ]; then
rake_tasks+=(apt:rc:artifactory apt:rc)
apt_targets+=(debian)
fi
-if [ ${UPLOAD_DOCS} -gt 0 ]; then
+if [ "${UPLOAD_DOCS}" -gt 0 ]; then
rake_tasks+=(docs:rc)
fi
-if [ ${UPLOAD_PYTHON} -gt 0 ]; then
- rake_tasks+=(python:rc)
-fi
-if [ ${UPLOAD_R} -gt 0 ]; then
+if [ "${UPLOAD_R}" -gt 0 ]; then
rake_tasks+=(r:rc)
fi
-if [ ${UPLOAD_UBUNTU} -gt 0 ]; then
+if [ "${UPLOAD_UBUNTU}" -gt 0 ]; then
rake_tasks+=(apt:rc:artifactory apt:rc)
apt_targets+=(ubuntu)
fi
rake_tasks+=(summary:rc)
-tmp_dir=binary/tmp
-mkdir -p "${tmp_dir}"
source_artifacts_dir="${tmp_dir}/artifacts"
rm -rf "${source_artifacts_dir}"
cp -a "${ARROW_ARTIFACTS_DIR}" "${source_artifacts_dir}"
@@ -119,17 +143,23 @@ cp -a "${ARROW_ARTIFACTS_DIR}" "${source_artifacts_dir}"
docker_run \
./runner.sh \
rake \
- "${rake_tasks[@]}" \
- APT_TARGETS=$(IFS=,; echo "${apt_targets[*]}") \
- ARTIFACTORY_API_KEY="${ARTIFACTORY_API_KEY}" \
- ARTIFACTS_DIR="${tmp_dir}/artifacts" \
- ASF_PASSWORD="${ASF_PASSWORD}" \
- ASF_USER="${ASF_USER}" \
- DEB_PACKAGE_NAME=${DEB_PACKAGE_NAME:-} \
- DRY_RUN=${DRY_RUN:-no} \
- GPG_KEY_ID="${GPG_KEY_ID}" \
- RC=${rc} \
- STAGING=${STAGING:-no} \
- VERBOSE=${VERBOSE:-no} \
- VERSION=${version} \
- YUM_TARGETS=$(IFS=,; echo "${yum_targets[*]}")
+ "${rake_tasks[@]}" \
+ APT_TARGETS="$(
+ IFS=,
+ echo "${apt_targets[*]}"
+ )" \
+ ARTIFACTORY_API_KEY="${ARTIFACTORY_API_KEY}" \
+ ARTIFACTS_DIR="${tmp_dir}/artifacts" \
+ ASF_PASSWORD="${ASF_PASSWORD}" \
+ ASF_USER="${ASF_USER}" \
+ DEB_PACKAGE_NAME="${DEB_PACKAGE_NAME:-}" \
+ DRY_RUN="${DRY_RUN:-no}" \
+ GPG_KEY_ID="${GPG_KEY_ID}" \
+ RC="${rc}" \
+ STAGING="${STAGING:-no}" \
+ VERBOSE="${VERBOSE:-no}" \
+ VERSION="${version}" \
+ YUM_TARGETS="$(
+ IFS=,
+ echo "${yum_targets[*]}"
+ )"
diff --git a/dev/release/binary-task.rb b/dev/release/binary-task.rb
index 8165885ab6..86056ec9d0 100644
--- a/dev/release/binary-task.rb
+++ b/dev/release/binary-task.rb
@@ -1124,7 +1124,6 @@ class BinaryTask
define_apt_tasks
define_yum_tasks
define_docs_tasks
- define_python_tasks
define_r_tasks
define_summary_tasks
end
@@ -2527,14 +2526,6 @@ APT::FTPArchive::Release::Description
"#{apt_repository_description}";
"test-debian-12-docs/**/*")
end
- def define_python_tasks
- define_generic_data_tasks("Python",
- :python,
- "#{rc_dir}/python/#{full_version}",
- "#{release_dir}/python/#{full_version}",
- "{python-sdist,wheel-*}/**/*")
- end
-
def define_r_rc_tasks(label, id, rc_dir)
directory rc_dir
diff --git a/dev/release/post-03-binary.sh b/dev/release/post-03-binary.sh
index 493521a174..4ca545885f 100755
--- a/dev/release/post-03-binary.sh
+++ b/dev/release/post-03-binary.sh
@@ -37,6 +37,7 @@ if [ ! -f .env ]; then
echo "You can use $(pwd)/.env.example as template"
exit 1
fi
+# shellcheck source=SCRIPTDIR/.env.example
. .env
. utils-binary.sh
@@ -44,45 +45,41 @@ fi
# By default deploy all artifacts.
# To deactivate one category, deactivate the category and all of its
dependents.
# To explicitly select one category, set DEPLOY_DEFAULT=0 DEPLOY_X=1.
-: ${DEPLOY_DEFAULT:=1}
-: ${DEPLOY_ALMALINUX:=${DEPLOY_DEFAULT}}
-: ${DEPLOY_AMAZON_LINUX:=${DEPLOY_DEFAULT}}
-: ${DEPLOY_CENTOS:=${DEPLOY_DEFAULT}}
-: ${DEPLOY_DEBIAN:=${DEPLOY_DEFAULT}}
-: ${DEPLOY_DOCS:=${DEPLOY_DEFAULT}}
-: ${DEPLOY_PYTHON:=${DEPLOY_DEFAULT}}
-: ${DEPLOY_R:=${DEPLOY_DEFAULT}}
-: ${DEPLOY_UBUNTU:=${DEPLOY_DEFAULT}}
+: "${DEPLOY_DEFAULT:=1}"
+: "${DEPLOY_ALMALINUX:=${DEPLOY_DEFAULT}}"
+: "${DEPLOY_AMAZON_LINUX:=${DEPLOY_DEFAULT}}"
+: "${DEPLOY_CENTOS:=${DEPLOY_DEFAULT}}"
+: "${DEPLOY_DEBIAN:=${DEPLOY_DEFAULT}}"
+: "${DEPLOY_DOCS:=${DEPLOY_DEFAULT}}"
+: "${DEPLOY_R:=${DEPLOY_DEFAULT}}"
+: "${DEPLOY_UBUNTU:=${DEPLOY_DEFAULT}}"
rake_tasks=()
apt_targets=()
yum_targets=()
-if [ ${DEPLOY_ALMALINUX} -gt 0 ]; then
+if [ "${DEPLOY_ALMALINUX}" -gt 0 ]; then
rake_tasks+=(yum:artifactory:release)
yum_targets+=(almalinux)
fi
-if [ ${DEPLOY_AMAZON_LINUX} -gt 0 ]; then
+if [ "${DEPLOY_AMAZON_LINUX}" -gt 0 ]; then
rake_tasks+=(yum:artifactory:release)
yum_targets+=(amazon-linux)
fi
-if [ ${DEPLOY_CENTOS} -gt 0 ]; then
+if [ "${DEPLOY_CENTOS}" -gt 0 ]; then
rake_tasks+=(yum:artifactory:release)
yum_targets+=(centos)
fi
-if [ ${DEPLOY_DEBIAN} -gt 0 ]; then
+if [ "${DEPLOY_DEBIAN}" -gt 0 ]; then
rake_tasks+=(apt:artifactory:release)
apt_targets+=(debian)
fi
-if [ ${DEPLOY_DOCS} -gt 0 ]; then
+if [ "${DEPLOY_DOCS}" -gt 0 ]; then
rake_tasks+=(docs:release)
fi
-if [ ${DEPLOY_PYTHON} -gt 0 ]; then
- rake_tasks+=(python:release)
-fi
-if [ ${DEPLOY_R} -gt 0 ]; then
+if [ "${DEPLOY_R}" -gt 0 ]; then
rake_tasks+=(r:release)
fi
-if [ ${DEPLOY_UBUNTU} -gt 0 ]; then
+if [ "${DEPLOY_UBUNTU}" -gt 0 ]; then
rake_tasks+=(apt:artifactory:release)
apt_targets+=(ubuntu)
fi
@@ -94,13 +91,19 @@ mkdir -p "${tmp_dir}"
docker_run \
./runner.sh \
rake \
- --trace \
- "${rake_tasks[@]}" \
- APT_TARGETS=$(IFS=,; echo "${apt_targets[*]}") \
- ARTIFACTORY_API_KEY="${ARTIFACTORY_API_KEY}" \
- ARTIFACTS_DIR="${tmp_dir}/artifacts" \
- RC=${rc} \
- STAGING=${STAGING:-no} \
- VERBOSE=${VERBOSE:-no} \
- VERSION=${version} \
- YUM_TARGETS=$(IFS=,; echo "${yum_targets[*]}")
+ --trace \
+ "${rake_tasks[@]}" \
+ APT_TARGETS="$(
+ IFS=,
+ echo "${apt_targets[*]}"
+ )" \
+ ARTIFACTORY_API_KEY="${ARTIFACTORY_API_KEY}" \
+ ARTIFACTS_DIR="${tmp_dir}/artifacts" \
+ RC="${rc}" \
+ STAGING="${STAGING:-no}" \
+ VERBOSE="${VERBOSE:-no}" \
+ VERSION="${version}" \
+ YUM_TARGETS="$(
+ IFS=,
+ echo "${yum_targets[*]}"
+ )"
diff --git a/dev/release/post-11-python.sh b/dev/release/post-11-python.sh
index 65ef44afa6..f94c229a42 100755
--- a/dev/release/post-11-python.sh
+++ b/dev/release/post-11-python.sh
@@ -21,7 +21,8 @@ set -ex
set -o pipefail
SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-: ${TEST_PYPI:=0}
+: "${GITHUB_REPOSITORY:=apache/arrow}"
+: "${TEST_PYPI:=0}"
if [ "$#" -ne 1 ]; then
echo "Usage: $0 <version>"
@@ -30,26 +31,29 @@ fi
version=$1
+cd "${SOURCE_DIR}"
+
+if [ ! -f .env ]; then
+ echo "You must create $(pwd)/.env"
+ echo "You can use $(pwd)/.env.example as template"
+ exit 1
+fi
+# shellcheck source=SCRIPTDIR/.env.example
+. .env
+
tmp=$(mktemp -d -t "arrow-post-python.XXXXX")
-base_url=https://apache.jfrog.io/artifactory/arrow/python/${version}
-curl \
- --location \
- ${base_url} | \
- grep -E -o "pyarrow-${version}[a-zA-Z0-9._-]*\\.(tar\\.gz|whl)" | \
- sort | \
- uniq | while read artifact; do
- curl \
- --fail \
- --location \
- --output ${tmp}/${artifact} \
- ${base_url}/${artifact}
-done
-
-if [ ${TEST_PYPI} -gt 0 ]; then
+gh release download \
+ "apache-arrow-${version}" \
+ --dir "${tmp}" \
+ --pattern "pyarrow-*.tar.gz" \
+ --pattern "pyarrow-*.whl" \
+ --repo "${GITHUB_REPOSITORY}"
+
+if [ "${TEST_PYPI}" -gt 0 ]; then
TWINE_ARGS="--repository-url https://test.pypi.org/legacy/";
fi
-twine upload ${TWINE_ARGS} ${tmp}/*
+twine upload "${TWINE_ARGS}" "${tmp}"/*
rm -rf "${tmp}"
