This is an automated email from the ASF dual-hosted git repository. lostluck pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new 7d281558dd8 [#29697] Add prism artifact building workflow. (#31369) 7d281558dd8 is described below commit 7d281558dd8e7900e4422612479a051776364aec Author: Robert Burke <lostl...@users.noreply.github.com> AuthorDate: Tue May 28 20:14:40 2024 -0700 [#29697] Add prism artifact building workflow. (#31369) --- .github/workflows/build_release_candidate.yml | 188 ++++++++++++++++++++++---- 1 file changed, 160 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml index b9e62c35aad..37a70fd2f8d 100644 --- a/.github/workflows/build_release_candidate.yml +++ b/.github/workflows/build_release_candidate.yml @@ -19,29 +19,35 @@ on: APACHE_PASSWORD: description: Your Apache password. Required if you want to stage artifacts into https://dist.apache.org/repos/dist/dev/beam/ required: false - BEAM_SITE_TOKEN: - description: Github Personal Access Token with apache/beam-site repo permission if you want to create the beam-site docs PR. See https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens. + REPO_TOKEN: + description: | + Github Personal Access Token with repo permissions if you want to create the beam-site docs PR, + create a draft release, or upload prism artifacts to that release. + See https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens. default: '' - PUBLISH_JAVA_ARTIFACTS: - description: Whether to publish java artifacts to https://repository.apache.org/#stagingRepositories (yes/no) + STAGE: + description: | + Configure which actions this workflow should perform, by setting the desired fields to "yes". + -- + java_artifacts: publish java artifacts to https://repository.apache.org/#stagingRepositories + -- + java_source: java source into https://dist.apache.org/repos/dist/dev/beam/ + -- + docker_artifacts: stage SDK docker images to docker hub Apache organization + -- + python_artifacts: stage the python artifacts into https://dist.apache.org/repos/dist/dev/beam/ + -- + beam_site_pr: create the documentation update PR against apache/beam-site. + -- + prism: build and upload the artifacts to the release for this tag required: true - default: 'no' - STAGE_JAVA_SOURCE: - description: Whether to stage the java source into https://dist.apache.org/repos/dist/dev/beam/ - required: true - default: 'no' - STAGE_DOCKER_ARTIFACTS: - description: Whether to stage SDK docker images to docker hub Apache organization - required: true - default: 'no' - STAGE_PYTHON_ARTIFACTS: - description: Whether to stage the python artifacts into https://dist.apache.org/repos/dist/dev/beam/ - required: true - default: 'no' - CREATE_BEAM_SITE_PR: - description: Whether to create the documentation update PR against apache/beam-site. - required: true - default: 'no' + default: | + {java_artifacts: "no", + java_source: "no", + docker_artifacts: "no", + python_artifacts: "no", + beam_site_pr: "no", + prism: "no"} env: GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} @@ -50,7 +56,7 @@ env: jobs: publish_java_artifacts: - if: ${{github.event.inputs.PUBLISH_JAVA_ARTIFACTS == 'yes'}} + if: ${{ fromJson(github.event.inputs.STAGE).java_artifacts == 'yes'}} runs-on: [self-hosted, ubuntu-20.04, main] steps: - name: Checkout @@ -92,7 +98,7 @@ jobs: stage_java_source: - if: ${{github.event.inputs.STAGE_JAVA_SOURCE == 'yes'}} + if: ${{ fromJson(github.event.inputs.STAGE).java_source == 'yes'}} runs-on: ubuntu-latest steps: - name: Mask Apache Password @@ -156,7 +162,7 @@ jobs: svn commit -m "Staging Java artifacts for Apache Beam ${{ github.event.inputs.RELEASE }} RC${{ github.event.inputs.RC }}" --non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password "${{ github.event.inputs.APACHE_PASSWORD }}" stage_python_artifacts: - if: ${{github.event.inputs.STAGE_PYTHON_ARTIFACTS == 'yes'}} + if: ${{ fromJson(github.event.inputs.STAGE).python_artifacts == 'yes'}} runs-on: ubuntu-latest steps: - name: Checkout @@ -239,7 +245,7 @@ jobs: stage_docker: - if: ${{github.event.inputs.STAGE_DOCKER_ARTIFACTS == 'yes'}} + if: ${{ fromJson(github.event.inputs.STAGE).docker_artifacts == 'yes'}} # Note: if this ever changes to self-hosted, remove the "Remove default github maven configuration" step runs-on: ubuntu-latest steps: @@ -283,7 +289,7 @@ jobs: run: ./gradlew :pushAllDockerImages -PisRelease -Pdocker-pull-licenses -Pprune-images -Pdocker-tag=${{ github.event.inputs.RELEASE }}rc${{ github.event.inputs.RC }} -Pjava11Home=${{steps.export-java11.outputs.JAVA11_HOME}} --no-daemon --no-parallel beam_site_pr: - if: ${{github.event.inputs.CREATE_BEAM_SITE_PR == 'yes'}} + if: ${{ fromJson(github.event.inputs.STAGE).beam_site_pr == 'yes'}} # Note: if this ever changes to self-hosted, remove the "Remove default github maven configuration" step runs-on: ubuntu-latest env: @@ -303,7 +309,7 @@ jobs: with: repository: apache/beam-site path: beam-site - token: ${{ github.event.inputs.BEAM_SITE_TOKEN }} + token: ${{ github.event.inputs.REPO_TOKEN }} ref: release-docs - name: Install Python 3.8 uses: actions/setup-python@v5 @@ -396,8 +402,134 @@ jobs: - name: Create beam-site PR working-directory: beam-site env: - GH_TOKEN: ${{ github.event.inputs.BEAM_SITE_TOKEN }} + GH_TOKEN: ${{ github.event.inputs.REPO_TOKEN }} PR_TITLE: "Publish docs for ${{ github.event.inputs.RELEASE }} release" PR_BODY: "Content generated from https://github.com/apache/beam/tree/${{ env.RC_TAG }}." run: | gh pr create -t "$PR_TITLE" -b "$PR_BODY" --base release-docs + + build_and_stage_prism: + if: ${{ fromJson(github.event.inputs.STAGE).prism == 'yes'}} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" + repository: apache/beam + - name: Mask Apache Password + run: | + # Workaround for Actions bug - https://github.com/actions/runner/issues/643 + APACHE_PASS=$(jq -r '.inputs.APACHE_PASSWORD' $GITHUB_EVENT_PATH) + echo "::add-mask::$APACHE_PASS" + - name: Mask apache id/password + run: | + if [ "${{ github.event.inputs.APACHE_ID }}" == "" ] + then + echo "Must provide an apache id to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/" + fi + if [ "${{ github.event.inputs.APACHE_PASSWORD }}" == "" ] + then + echo "Must provide an apache password to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/" + fi + - uses: actions/setup-go@v5 + with: + go-version: '1.22' + cache-dependency-path: | + sdks/go.sum + - name: Import GPG key + id: import_gpg + uses: crazy-max/ghaction-import-gpg@111c56156bcc6918c056dbef52164cfa583dc549 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + - name: Build prism artifacts + env: + RC_TAG: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" + GIT_REPO_BASE_URL: https://github.com/apache/beam + RELEASE_DIR: "beam/${{ github.event.inputs.RELEASE }}" + RELEASE: "${{ github.event.inputs.RELEASE }}" + GH_TOKEN: ${{ github.event.inputs.REPO_TOKEN }} + SVN_ARTIFACTS_DIR: "beam/${{ github.event.inputs.RELEASE }}/prism" + run: | + # TODO stop the action if a non-prerelease release exists + # TODO create the draft release automatically if none exist. + # TODO update the draft release's RC tag to current. + + export GH_RELEASE_ID=`gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/apache/beam/releases | jq '.[] | select(.tag_name | contains(env.RC_TAG)) .id'` + + echo "with gh release id $GH_RELEASE_ID for tag $RC_TAG" + + # Store a reference so we can get back to the start. + ROOT_DIR=`pwd` + echo "ROOT_DIR=$ROOT_DIR" + echo "............Checking out Apache Dev Repo.........." + + svn co https://dist.apache.org/repos/dist/dev/beam + mkdir -p "${SVN_ARTIFACTS_DIR}" + + OUTPUT_DIR=$ROOT_DIR/$SVN_ARTIFACTS_DIR + mkdir -p $OUTPUT_DIR + echo "OUTPUT_DIR=$OUTPUT_DIR" + + # Remove existing prism contents from the svn repo. + cd $OUTPUT_DIR + rm -rf * + cd $ROOT_DIR + + # Navigate to the prism command directory. + cd sdks/go/cmd/prism + + # Store a reference to where we're building from. + BUILD_DIR=`pwd` + + echo "............Build and stage prism artifacts in the Github Release $GH_RELEASE_ID for tag $RC_TAG.........." + + + # Loop through and build desired set from allowed types. + for OS in linux windows darwin; do + for ARCH in amd64 arm64; do + TARGET_DIR="$OUTPUT_DIR/$OS/$ARCH" + TARGET_NAME="apache_beam-v$RELEASE-prism-$OS-$ARCH" + + mkdir -p "$TARGET_DIR" + OUTPUT_FILE="$TARGET_DIR/$TARGET_NAME" + if GOOS=$OS GOARCH=$ARCH CGO_ENABLED=0 go build -trimpath -buildvcs=false -o "$OUTPUT_FILE" . > /dev/null 2>&1; then + cd $TARGET_DIR + # Extract real output name. Windows builds automatically have .exe added. + ARTIFACT=`ls` + echo "target built - $ARTIFACT" + + ZIP_NAME="$ARTIFACT.zip" + zip -r $ZIP_NAME $ARTIFACT + + # Sign and hash + gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --batch --yes --detach-sig $ZIP_NAME + sha512sum $ZIP_NAME > "${ZIP_NAME}.sha512" + + # Upload to the release. + gh release upload $RC_TAG $ZIP_NAME ${ZIP_NAME}.sha512 ${ZIP_NAME}.asc --clobber + + # Remove the binary from the release directory + rm $ARTIFACT + + # Return to our root build dir. + cd $BUILD_DIR + else + echo "target failed for $OS - $ARCH" + rm -f "$OUTPUT_FILE" + exit 1 # Do not continue action + fi + done + done + echo "...........Prism builds done!..........." + cd $ROOT_DIR + cd $RELEASE_DIR + + echo "...........Adding prism artifacts to the Dev Apache SVN repo..........." + + svn add --force prism + svn status + svn commit -m "Staging Prism artifacts for Apache Beam ${RELEASE} RC${RC_NUM}" --non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password "${{ github.event.inputs.APACHE_PASSWORD }}"