This is an automated email from the ASF dual-hosted git repository. pabloem pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new 86ae0857fbc [Playground] Remove old deployment workflows (#27027) 86ae0857fbc is described below commit 86ae0857fbcecdaff9c24d6d6d87753168f89ac3 Author: Timur Sultanov <timur.sulta...@akvelon.com> AuthorDate: Mon Jun 12 23:54:58 2023 +0400 [Playground] Remove old deployment workflows (#27027) * [Playground] Remove old deployment workflows These workflows had been replaced by new CloudBuild triggers based ones. * Remove doc for non-existing workflows --- .github/workflows/build_playground_backend.yml | 98 ---------- .github/workflows/build_playground_frontend.yml | 92 ---------- .github/workflows/playground_deploy_backend.yml | 91 ---------- .github/workflows/playground_deploy_examples.yml | 198 -------------------- .../workflows/playground_deploy_infrastructure.yml | 33 ---- .github/workflows/playground_examples_cd.yml | 36 ---- .../workflows/playground_examples_cd_reusable.yml | 77 -------- .github/workflows/playground_examples_ci.yml | 51 ------ .../workflows/playground_examples_ci_reusable.yml | 201 --------------------- playground/CI.md | 118 ------------ 10 files changed, 995 deletions(-) diff --git a/.github/workflows/build_playground_backend.yml b/.github/workflows/build_playground_backend.yml deleted file mode 100644 index 137fe3cf6ab..00000000000 --- a/.github/workflows/build_playground_backend.yml +++ /dev/null @@ -1,98 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Build And Deploy Playground Backend Application - -on: - push: - tags: ['v*'] - branches: ['master', 'release-*'] - pull_request: - paths: ['playground/backend/**'] - branches: ['playground-staging'] - workflow_dispatch: -# This allows a subsequently queued workflow run to interrupt previous runs -concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' - cancel-in-progress: true - -jobs: - build_playground_backend_docker_image: - name: Build Playground Backend App - runs-on: [self-hosted, ubuntu-20.04] - env: - GO_VERSION: 1.19.3 - BEAM_VERSION: 2.40.0 - TERRAFORM_VERSION: 1.0.9 - STAND_SUFFIX: '' - DATASTORE_EMULATOR_HOST: 127.0.0.1:8888 - steps: - - name: Check out the repo - uses: actions/checkout@v3 - - name: Set up Java - uses: actions/setup-java@v3.8.0 - with: - distribution: 'zulu' - java-version: '8' - - uses: actions/setup-go@v3 - with: - go-version: '${{ env.GO_VERSION }}' - - name: Install dependencies - run: sudo apt update && sudo apt install libsnmp-dev -y - - name: Remove default github maven configuration - # This step is a workaround to avoid a decryption issue - run: rm ~/.m2/settings.xml - - name: Install sbt for running SCIO tests - run: | - echo "deb https://repo.scala-sbt.org/scalasbt/debian all main" | sudo tee /etc/apt/sources.list.d/sbt.list &&\ - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | sudo tee /etc/apt/sources.list.d/sbt_old.list &&\ - curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823" | sudo apt-key add - sudo apt-get update && sudo apt-get install -y sbt - - name: Set up Cloud SDK and its components - uses: google-github-actions/setup-gcloud@v0 - with: - install_components: 'beta,cloud-datastore-emulator' - version: '389.0.0' - - - name: Run PreCommit - run: ./gradlew playground:backend:precommit - - uses: hashicorp/setup-terraform@v1 - with: - terraform_version: ${{ env.TERRAFORM_VERSION }} - if: startsWith(github.ref, 'refs/tags/') - - name: Set Docker Tag - run: echo "DOCKERTAG=${GITHUB_SHA}" >> $GITHUB_ENV - - name: Set Docker Tag If Github Tag was trigger - run: echo "DOCKERTAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - if: startsWith(github.ref, 'refs/tags/') - - name: Setup GCP account - run: | - echo "${{ secrets.GCP_PLAYGROUND_SA_KEY }}" | base64 -d > /tmp/gcp_access.json - if: startsWith(github.ref, 'refs/tags/') - - name: Login to Docker Registry - run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.PLAYGROUND_REGISTRY_NAME }} - if: startsWith(github.ref, 'refs/tags/') - - name: Deploy Backend Applications - env: - GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json - run: ./gradlew playground:terraform:deployBackend -Pdocker-tag=${{env.DOCKERTAG}} -Pproject_id=${{ secrets.GCP_PLAYGROUND_PROJECT_ID }} -Pproject_environment='beta' -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.GCP_PLAYGROUND_PROJECT_ID }}/playground-repository' - if: startsWith(github.ref, 'refs/tags/') - - name: DB Index creation - run: | - gcloud auth activate-service-account --project=${{ secrets.GCP_PLAYGROUND_PROJECT_ID }} --key-file=/tmp/gcp_access.json - gcloud app deploy playground/index.yaml --project=${{ secrets.GCP_PLAYGROUND_PROJECT_ID }} - if: startsWith(github.ref, 'refs/tags/') - env: - GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json \ No newline at end of file diff --git a/.github/workflows/build_playground_frontend.yml b/.github/workflows/build_playground_frontend.yml deleted file mode 100644 index 27f998ae224..00000000000 --- a/.github/workflows/build_playground_frontend.yml +++ /dev/null @@ -1,92 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Build And Deploy Playground Frontend Application - -on: - push: - tags: ['v*'] - branches: ['master', 'release-*'] - pull_request: - paths: ['playground/backend/**'] - branches: ['playground-staging'] - workflow_dispatch: - -# This allows a subsequently queued workflow run to interrupt previous runs -concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' - cancel-in-progress: true - -jobs: - build_playground_frontend_docker_image: - name: Build Playground Frontend App - runs-on: [self-hosted, ubuntu-20.04] - env: - GO_VERSION: 1.19.6 - BEAM_VERSION: 2.40.0 - TERRAFORM_VERSION: 1.0.9 - FLUTTER_VERSION: 3.10.2 - STAND_SUFFIX: '' - GOOGLE_DOMAIN: '-dot-apache-beam-testing.appspot.com' - steps: - - name: Check out the repo - uses: actions/checkout@v3 - - name: Set up Java - uses: actions/setup-java@v3.8.0 - with: - distribution: 'zulu' - java-version: '8' - - name: Install Flutter - run: | - wget https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_$FLUTTER_VERSION-stable.tar.xz &&\ - tar -xf flutter_linux_$FLUTTER_VERSION-stable.tar.xz &&\ - sudo mv flutter /opt/ &&\ - sudo ln -s /opt/flutter/bin/flutter /usr/local/bin/flutter &&\ - sudo ln -s /opt/flutter/bin/dart /usr/local/bin/dart &&\ - sudo dart pub global activate protoc_plugin &&\ - sudo ln -s /root/.pub-cache/bin/protoc-gen-dart /usr/local/bin/ - - name: Remove default github maven configuration - # This step is a workaround to avoid a decryption issue - run: sudo rm ~/.m2/settings.xml - - name: install npm - uses: actions/setup-node@v3 - with: - node-version: '14' - - name: install docker linter - run: npm install -g dockerlint - - name: lint dockerfile - run: dockerlint Dockerfile - working-directory: playground/frontend - - uses: hashicorp/setup-terraform@v1 - with: - terraform_version: ${{ env.TERRAFORM_VERSION }} - if: startsWith(github.ref, 'ref/tags/') - - name: Set Docker Tag - run: echo "DOCKERTAG=${GITHUB_SHA}" >> $GITHUB_ENV - - name: Set Docker Tag If Github Tag was trigger - run: echo "DOCKERTAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - if: startsWith(github.ref, 'refs/tags/') - - name: Setup GCP account - run: | - echo "${{ secrets.GCP_PLAYGROUND_SA_KEY }}" | base64 -d > /tmp/gcp_access.json - if: startsWith(github.ref, 'ref/tags/') - - name: Login to Docker Registry - run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.PLAYGROUND_REGISTRY_NAME }} - if: startsWith(github.ref, 'ref/tags/') - - name: Deploy Frontend Application - env: - GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json - run: ./gradlew playground:terraform:deployFrontend -Pdocker-tag=${{env.DOCKERTAG}} -Pproject_id=${{ secrets.GCP_PLAYGROUND_PROJECT_ID }} -Pproject_environment='beta' - if: startsWith(github.ref, 'refs/tags/') \ No newline at end of file diff --git a/.github/workflows/playground_deploy_backend.yml b/.github/workflows/playground_deploy_backend.yml deleted file mode 100644 index 02a4c799261..00000000000 --- a/.github/workflows/playground_deploy_backend.yml +++ /dev/null @@ -1,91 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Build And Deploy Playground Backend Application - -on: - push: - tags: 'v*' - branches: ['master', 'release-*'] - pull_request: - paths: ['playground/backend/**'] - branches: ['playground-staging'] - workflow_dispatch: - -# This allows a subsequently queued workflow run to interrupt previous runs -concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' - cancel-in-progress: true - -jobs: - build_playground_backend_docker_image: - name: Build Playground Backend App - runs-on: ubuntu-latest - env: - GO_VERSION: 1.19.6 - BEAM_VERSION: 2.40.0 - TERRAFORM_VERSION: 1.0.9 - STAND_SUFFIX: '' - DATASTORE_EMULATOR_HOST: 127.0.0.1:8888 - steps: - - name: Check out the repo - uses: actions/checkout@v3 - - uses: actions/setup-java@v3.8.0 - with: - distribution: 'zulu' - java-version: '8' - - uses: actions/setup-go@v3 - with: - go-version: '${{ env.GO_VERSION }}' - - name: Remove default github maven configuration - # This step is a workaround to avoid a decryption issue - run: rm ~/.m2/settings.xml - - - name: Set up Cloud SDK and its components - uses: google-github-actions/setup-gcloud@v0 - with: - install_components: 'beta,cloud-datastore-emulator' - version: '389.0.0' - - - name: Run PreCommit - run: ./gradlew playground:backend:precommit - - uses: hashicorp/setup-terraform@v1 - with: - terraform_version: ${{ env.TERRAFORM_VERSION }} - if: startsWith(github.ref, 'refs/tags/') - - name: Set Docker Tag - run: echo "DOCKERTAG=${GITHUB_SHA}" >> $GITHUB_ENV - - name: Set Docker Tag If Github Tag was trigger - run: echo "DOCKERTAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - if: startsWith(github.ref, 'refs/tags/') - - name: Setup GCP account - run: | - echo "${{ secrets.GCP_PLAYGROUND_SA_KEY }}" | base64 -d > /tmp/gcp_access.json - if: startsWith(github.ref, 'refs/tags/') - - name: Login to Docker Registry - run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.PLAYGROUND_REGISTRY_NAME }} - if: startsWith(github.ref, 'refs/tags/') - - name: Deploy Backend Applications - env: - GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json - run: ./gradlew playground:terraform:deployBackend -Pdocker-tag=${{env.DOCKERTAG}} -Pproject_id=${{ secrets.GCP_PLAYGROUND_PROJECT_ID }} -Pproject_environment='beta' -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.GCP_PLAYGROUND_PROJECT_ID }}/playground-repository' - if: startsWith(github.ref, 'refs/tags/') - - name: DB Index creation - run: | - gcloud auth activate-service-account --project=${{ secrets.GCP_PLAYGROUND_PROJECT_ID }} --key-file=/tmp/gcp_access.json - gcloud app deploy playground/index.yaml --project=${{ secrets.GCP_PLAYGROUND_PROJECT_ID }} - if: startsWith(github.ref, 'refs/tags/') - env: - GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json diff --git a/.github/workflows/playground_deploy_examples.yml b/.github/workflows/playground_deploy_examples.yml deleted file mode 100644 index cede42c8f88..00000000000 --- a/.github/workflows/playground_deploy_examples.yml +++ /dev/null @@ -1,198 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Collect And Deploy Playground Examples - -on: - workflow_dispatch: -env: - BEAM_ROOT_DIR: "../../" - SDK_CONFIG: "../../playground/sdks.yaml" - BEAM_EXAMPLE_CATEGORIES: "../categories.yaml" - BEAM_VERSION: 2.40.0 - K8S_NAMESPACE: playground-backend - HELM_APP_NAME: playground-backend - SUBDIRS: "./learning/katas ./examples ./sdks" - ORIGIN: PG_EXAMPLES -jobs: - check_examples: - name: Check examples - runs-on: ubuntu-latest - outputs: - example_has_changed: ${{ steps.check_has_example.outputs.example_has_changed }} - steps: - - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - name: install deps - run: pip install -r requirements.txt - working-directory: playground/infrastructure - - name: get Difference - id: check_file_changed - run: | - set -xeu - # define the base ref - BASE_REF=$GITHUB_BASE_REF - if [ -z "$BASE_REF" ] || [ "$BASE_REF" == "master" ]; then - BASE_REF=origin/master - fi - DIFF=$(git diff --name-only $BASE_REF $GITHUB_SHA -- $SUBDIRS) - echo "example_diff=$DIFF" >> $GITHUB_OUTPUT - - name: has Examples - run: | - output=$(python3 checker.py ${{ steps.check_file_changed.outputs.example_diff }}) - echo "example_has_changed=$output" >> $GITHUB_OUTPUT - id: check_has_example - working-directory: playground/infrastructure - - name: Print has_example - run: echo "${{ steps.check_has_example.outputs.example_has_changed }}" - deploy_examples: - name: Deploy examples - runs-on: ubuntu-latest - needs: [ check_examples ] - if: needs.check_examples.outputs.example_has_changed == 'True' - env: - GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json - GOOGLE_CLOUD_PROJECT: ${{ secrets.GCP_PLAYGROUND_PROJECT_ID }} - SA_KEY_CONTENT: ${{ secrets.GCP_PLAYGROUND_SA_KEY }} - REGION: us-central1 - steps: - - name: Check out the repo - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - uses: actions/setup-java@v3.8.0 - with: - distribution: 'zulu' - java-version: '8' - - name: Install kubectl - run: | - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" &&\ - chmod +x kubectl &&\ - mv kubectl /usr/local/bin/ - - name: Install helm - run: | - curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 &&\ - chmod 700 get_helm.sh &&\ - ./get_helm.sh - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0 - with: - install_components: 'gke-gcloud-auth-plugin' - - name: install deps - run: pip install -r requirements.txt - working-directory: playground/infrastructure - - name: Remove default github maven configuration - # This step is a workaround to avoid a decryption issue - run: rm ~/.m2/settings.xml - - name: Setup GCP account - run: | - echo "$SA_KEY_CONTENT" | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS - which gcloud - gcloud auth activate-service-account --project=$GOOGLE_CLOUD_PROJECT \ - --key-file=$GOOGLE_APPLICATION_CREDENTIALS - - name: Set Docker Tag - run: echo "DOCKERTAG=${GITHUB_SHA}" >> $GITHUB_ENV - - name: Set Docker Tag If Github Tag was trigger - run: echo "DOCKERTAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - if: startsWith(github.ref, 'refs/tags/') - - name: Set Docker Registry env - run: echo "DOCKER_REGISTRY=$DOCKER_REGISTRY" >> $GITHUB_ENV - env: - DOCKER_REGISTRY: ${{ env.REGION }}-docker.pkg.dev/${{ env.GOOGLE_CLOUD_PROJECT }}/playground-repository - - - name: Get K8s Config - run: gcloud container clusters get-credentials --region us-central1-a playground-examples - - name: Login to Docker Registry - run: cat $GOOGLE_APPLICATION_CREDENTIALS | docker login -u _json_key --password-stdin "$PLAYGROUND_REGISTRY_URL" - env: - PLAYGROUND_REGISTRY_URL: https://${{ env.REGION }}-docker.pkg.dev - - name: Build And Push Java Backend - run: | - ./gradlew playground:backend:containers:java:dockerTagPush \ - -Pdocker-repository-root="$DOCKER_REGISTRY" \ - -Pbase-image="apache/beam_java8_sdk:$BEAM_VERSION" \ - -Pdocker-tag="$DOCKERTAG" - - name: Build And Push Go Backend - run: | - ./gradlew playground:backend:containers:go:dockerTagPush \ - -Pdocker-repository-root="$DOCKER_REGISTRY" \ - -Pdocker-tag="$DOCKERTAG" - - name: Build And Push Python Backend - run: | - ./gradlew playground:backend:containers:python:dockerTagPush \ - -Pdocker-repository-root="$DOCKER_REGISTRY" \ - -Pdocker-tag="$DOCKERTAG" - - name: Clean All Build directories - run: ./gradlew clean - - name: Install helm chart - run: | - kubectl create namespace $K8S_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - &&\ - helm install --namespace $K8S_NAMESPACE $HELM_APP_NAME . \ - --set global.registry="$DOCKER_REGISTRY" \ - --set global.tag="$DOCKERTAG" &&\ - sleep 120 - working-directory: playground/infrastructure/helm - - - name: Run Python Examples CI - run: | - K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-python" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}') - echo $K8S_SERVER_ADDRESS - export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8081" - python3 ci_cd.py --datastore-project $GOOGLE_CLOUD_PROJECT --step CI --sdk SDK_PYTHON --origin $ORIGIN --subdirs $SUBDIRS - working-directory: playground/infrastructure - - - name: Run Python Examples CD - run: | - K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-python" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}') - export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8081" - python3 ci_cd.py --datastore-project $GOOGLE_CLOUD_PROJECT --step CD --sdk SDK_PYTHON --origin $ORIGIN --subdirs $SUBDIRS - working-directory: playground/infrastructure - - - name: Run Go Examples CI - run: | - K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-go" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}') - export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8082" - python3 ci_cd.py --datastore-project $GOOGLE_CLOUD_PROJECT --step CI --sdk SDK_GO --origin $ORIGIN --subdirs $SUBDIRS - working-directory: playground/infrastructure - - - name: Run Go Examples CD - run: | - K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-go" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}') - export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8082" - python3 ci_cd.py --datastore-project $GOOGLE_CLOUD_PROJECT --step CD --sdk SDK_GO --origin $ORIGIN --subdirs $SUBDIRS - working-directory: playground/infrastructure - - - name: Run Java Examples CI - run: | - K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-java" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}') - export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8080" - python3 ci_cd.py --datastore-project $GOOGLE_CLOUD_PROJECT --step CI --sdk SDK_JAVA --origin $ORIGIN --subdirs $SUBDIRS - working-directory: playground/infrastructure - - name: Run Java Examples CD - run: | - K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-java" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}') - export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8080" - python3 ci_cd.py --datastore-project $GOOGLE_CLOUD_PROJECT --step CD --sdk SDK_JAVA --origin $ORIGIN --subdirs $SUBDIRS - working-directory: playground/infrastructure - - name: Delete Helm Chart - if: always() - run: | - helm del --namespace $K8S_NAMESPACE $HELM_APP_NAME \ No newline at end of file diff --git a/.github/workflows/playground_deploy_infrastructure.yml b/.github/workflows/playground_deploy_infrastructure.yml deleted file mode 100644 index 2f2e6b18d72..00000000000 --- a/.github/workflows/playground_deploy_infrastructure.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: New environment creation -on: - workflow_dispatch: -jobs: - Infrastructure_deployment: - runs-on: ubuntu-latest - env: - PROJECT_ENV: beta - steps: - - name: git clone repo - uses: actions/checkout@v2 - - name: Setup GCP account - run: | - echo "${{ secrets.GCP_PLAYGROUND_SA_KEY }}" | base64 -d > /tmp/gcp_access.json - - name: Deploy infrastructure - env: - GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json - run: ./gradlew playground:terraform:InitInfrastructure -Pproject_environment="${{ env.PROJECT_ENV }}" diff --git a/.github/workflows/playground_examples_cd.yml b/.github/workflows/playground_examples_cd.yml deleted file mode 100644 index c028451a44b..00000000000 --- a/.github/workflows/playground_examples_cd.yml +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Playground Examples CD - -on: - workflow_dispatch: -# Concurrency group for all deployment ops -concurrency: playground_production -jobs: - deploy_examples: - strategy: - matrix: - sdk: ["python", "go", "java"] - # run sequentially - max-parallel: 1 - uses: ./.github/workflows/playground_examples_cd_reusable.yml - with: - sdk: ${{ matrix.sdk }} - origin: PG_EXAMPLES - subdirs: "./learning/katas ./examples ./sdks" - secrets: - project_id: ${{ secrets.GCP_PLAYGROUND_PROJECT_ID }} - sa_key_content: ${{ secrets.GCP_PLAYGROUND_SA_KEY }} diff --git a/.github/workflows/playground_examples_cd_reusable.yml b/.github/workflows/playground_examples_cd_reusable.yml deleted file mode 100644 index 4c5b42e70e2..00000000000 --- a/.github/workflows/playground_examples_cd_reusable.yml +++ /dev/null @@ -1,77 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Playground Examples CD for a given SDK and origin - -on: - workflow_call: - inputs: - sdk: - type: string - required: true - subdirs: - type: string - required: true - origin: - type: string - required: true - secrets: - project_id: - required: true - sa_key_content: - required: true - -jobs: - cd: - name: CD ${{ inputs.sdk }} ${{ inputs.origin }} - runs-on: ubuntu-latest - env: - ORIGIN: ${{ inputs.origin }} - SDK: ${{ inputs.sdk }} - STEP: CD - SUBDIRS: ${{ inputs.subdirs }} - - GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json - GOOGLE_CLOUD_PROJECT: ${{ secrets.project_id }} - SA_KEY_CONTENT: ${{ secrets.sa_key_content }} - steps: - - name: Check out the repo - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - name: install deps - run: pip install -r requirements.txt - working-directory: playground/infrastructure - - name: Decode GCP credentials - run: | - echo "$SA_KEY_CONTENT" | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS - - - name: Run ci_cd.py - run: | - python3 ci_cd.py \ - --datastore-project $GOOGLE_CLOUD_PROJECT \ - --step $STEP \ - --sdk SDK_${SDK^^} \ - --origin $ORIGIN \ - --subdirs $SUBDIRS - working-directory: playground/infrastructure - env: - BEAM_ROOT_DIR: "../../" - SDK_CONFIG: "../../playground/sdks.yaml" - BEAM_EXAMPLE_CATEGORIES: "../categories.yaml" - SERVER_ADDRESS: https://backend-${{ env.SDK }}-beta-dot-apache-beam-testing.appspot.com - BEAM_USE_WEBGRPC: yes - BEAM_CONCURRENCY: 4 diff --git a/.github/workflows/playground_examples_ci.yml b/.github/workflows/playground_examples_ci.yml deleted file mode 100644 index 2dbd6a8835b..00000000000 --- a/.github/workflows/playground_examples_ci.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Playground Examples CI - -on: - workflow_dispatch: - pull_request: - paths: - - .github/workflows/playground_examples_ci_reusable.yml - - .github/workflows/playground_examples_ci.yml - - playground/backend/** - - playground/infrastructure/** - - learning/katas/** - - examples/** - - sdks/** -# This allows a subsequently queued workflow run to interrupt previous runs -concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' - cancel-in-progress: true -jobs: - check: - strategy: - matrix: - sdk: ["java", "python", "go"] - # run sequentially - max-parallel: 1 - uses: ./.github/workflows/playground_examples_ci_reusable.yml - with: - sdk: ${{ matrix.sdk }} - step: CI - origin: PG_EXAMPLES - subdirs: ./learning/katas ./examples ./sdks - # unfortunately, there's no input type for list - allowlist: | - .github/workflows/playground_examples_ci_reusable.yml \ - .github/workflows/playground_examples_ci.yml \ - playground/backend \ - playground/infrastructure \ diff --git a/.github/workflows/playground_examples_ci_reusable.yml b/.github/workflows/playground_examples_ci_reusable.yml deleted file mode 100644 index 702d89e1fe2..00000000000 --- a/.github/workflows/playground_examples_ci_reusable.yml +++ /dev/null @@ -1,201 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Playground Examples CI/CD for a given SDK and origin - -on: - workflow_call: - inputs: - step: - type: string - required: true - sdk: - type: string - required: true - subdirs: - type: string - required: true - allowlist: - type: string - required: true - origin: - type: string - required: true -env: - BEAM_VERSION: 2.44.0 -jobs: - check_has_examples: - name: pre-check - runs-on: ubuntu-latest - outputs: - example_has_changed: ${{ steps.check_has_example.outputs.example_has_changed }} - env: - SDK: ${{ inputs.sdk }} - steps: - - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - name: install deps - run: pip install -r requirements.txt - working-directory: playground/infrastructure - - name: pytest - run: pytest - working-directory: playground/infrastructure - - - name: get Difference - id: check_file_changed - run: | - set -xeu - # define the base ref - BASE_REF=$GITHUB_BASE_REF - if [ -z "$BASE_REF" ] || [ "$BASE_REF" == "master" ]; then - BASE_REF=origin/master - fi - DIFF=$(git diff --name-only $BASE_REF $GITHUB_SHA | tr '\n' ' ') - echo "example_diff=$DIFF" >> $GITHUB_OUTPUT - - id: check_has_example - name: has Examples - run: | - # don't quit on errors, check unbound vars, and show commands - set +e -ux - python3 checker.py \ - --verbose \ - --sdk SDK_${SDK^^} \ - --allowlist ${{ inputs.allowlist }} \ - --paths ${{ steps.check_file_changed.outputs.example_diff }} - CODE=$? - if [ $CODE -eq 0 ]; then - echo "example_has_changed=True" >> $GITHUB_OUTPUT - elif [ $CODE -eq 11 ]; then - echo "example_has_changed=False" >> $GITHUB_OUTPUT - else - echo "checker is broken" - exit 1 - fi - working-directory: playground/infrastructure - env: - BEAM_ROOT_DIR: "../.." - BEAM_EXAMPLE_CATEGORIES: "../categories.yaml" - - ci_cd: - name: ${{ inputs.step }} ${{ inputs.sdk }} ${{ inputs.origin }} - needs: [ check_has_examples ] - if: needs.check_has_examples.outputs.example_has_changed == 'True' - runs-on: ubuntu-latest - env: - ORIGIN: ${{ inputs.origin }} - SDK: ${{ inputs.sdk }} - STEP: ${{ inputs.step }} - SUBDIRS: ${{ inputs.subdirs }} - steps: - - name: Check out the repo - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - uses: actions/setup-java@v3.8.0 - with: - distribution: 'zulu' - java-version: '11' - - - name: install deps - run: pip install -r requirements.txt - working-directory: playground/infrastructure - - - name: Remove default github maven configuration - # This step is a workaround to avoid a decryption issue - run: rm ~/.m2/settings.xml - - - name: Set Docker Tag - run: echo "DOCKERTAG=${GITHUB_SHA}" >> $GITHUB_ENV - - name: Set Docker Tag If Github Tag was trigger - run: echo "DOCKERTAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - if: startsWith(github.ref, 'refs/tags/') - - - name: Setup Gradle - uses: gradle/gradle-build-action@v2 - with: - cache-read-only: false - - - name: Build an SDK runner base image and set SDK_TAG if needed - run: | - set -uex - # TODO make this a part of playground:backend:containers:python:docker task - if [ "$SDK" == "python" ]; then - # builds apache/beam_python3.10_sdk:$DOCKERTAG image - ./gradlew -i :sdks:python:container:py310:docker -Pdocker-tag="$DOCKERTAG" -Pdocker-pull-licenses=true - # and set SDK_TAG to DOCKERTAG so that the next step would find it - echo "SDK_TAG=${DOCKERTAG}" >> $GITHUB_ENV - fi - - - name: Build SDK Backend Docker image - run: | - set -ex - opts=" -Pdocker-tag=$DOCKERTAG" - if [ -n "$SDK_TAG" ]; then - opts="$opts -Psdk-tag=$SDK_TAG" - fi - if [ "$SDK" == "java" ]; then - # Java uses a fixed BEAM_VERSION - opts="$opts -Psdk-tag=$BEAM_VERSION" - fi - - # by default (w/o -Psdk-tag) runner uses BEAM from local ./sdks - # TODO Java SDK doesn't, it uses 2.44.0, fix this - ./gradlew -i playground:backend:containers:$SDK:docker $opts - - - name: Set docker image - run: echo "IMAGE_TAG=apache/beam_playground-backend-$SDK:$DOCKERTAG" >> $GITHUB_ENV - - - name: Start SDK runner in background - run: | - set -uex - NAME=$(docker run -d --rm -p 8080:8080 -e PROTOCOL_TYPE=TCP $IMAGE_TAG) - echo "NAME=$NAME" >> $GITHUB_ENV - - - name: Run ci_cd.py - run: | - python3 ci_cd.py \ - --step $STEP \ - --sdk SDK_${SDK^^} \ - --origin $ORIGIN \ - --subdirs $SUBDIRS - working-directory: playground/infrastructure - env: - BEAM_ROOT_DIR: "../../" - SDK_CONFIG: "../../playground/sdks.yaml" - BEAM_EXAMPLE_CATEGORIES: "../categories.yaml" - SERVER_ADDRESS: localhost:8080 - BEAM_CONCURRENCY: 4 - - - name: Get SDK runner logs - if: ${{ always() }} - run: | - [ -n "$NAME" ] && docker logs $NAME - - - name: Stop/remove SDK runner - if: ${{ always() }} - run: | - [ -n "$NAME" ] && docker rm -f $NAME - - - name: Delete Docker image - if: ${{ always() }} - run: | - docker image rm -f $IMAGE_TAG diff --git a/playground/CI.md b/playground/CI.md deleted file mode 100644 index 232fb9e850f..00000000000 --- a/playground/CI.md +++ /dev/null @@ -1,118 +0,0 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> - -# Apache Beam Playground - -## GitHub Actions - -### Pull request run - -Those runs are results of PR from the forks made by contributors. Most builds for Apache Beam fall -into this category. They are executed in the context of the "Fork", not main -Beam Code Repository which means that they have only "read" permission to all the GitHub resources -(container registry, code repository). This is necessary as the code in those PRs (including CI job -definition) might be modified by people who are not committers for the Apache Beam Code Repository. - -The main purpose of those jobs is to check if PR builds cleanly, if the test run properly and if -the PR is ready to review and merge. - -### Direct Push/Merge Run - -Those runs are results of direct pushes done by the committers or as result of merge of a Pull Request -by the committers. Those runs execute in the context of the Apache Beam Code Repository and have also -write permission for GitHub resources (container registry, code repository). -The main purpose for the run is to check if the code after merge still holds all the assertions - like -whether it still builds, all tests are green. - -This is needed because some of the conflicting changes from multiple PRs might cause build and test failures -after merge even if they do not fail in isolation. - -### Workflows - -#### Build And Deploy Playground Backend Application - [build_playground_backend.yml](.github/workflows/build_playground_backend.yml) - -| Job | Description | Pull Request Run | Direct Push/Merge Run | Requires GCP Credentials | -|-------------------------------------|---------------------------------------------------------------------|------------------|-----------------------|--------------------------| -| Check out the repo | GitHub Action used to check-out a repository. | Yes | Yes | No | -| setup-java | Install Java. | Yes | Yes | No | -| setup-go | Install Go. | Yes | Yes | No | -| maven config clean | Clean maven settings | Yes | Yes | No | -| set up cloud sdk and its components | Setting up cloud client and its components for tests | Yes | Yes | No | -| playground:backend:precommit | Pre commit playground. | Yes | Yes | No | -| npm install | Install nmp package. | Yes | Yes | No | -| lint dockerfile | Install and lint docker file. | Yes | Yes | No | -| setup-terraform | Install terraform. | Yes | Yes | No | -| Docker Tag | Add tag , if it not set. | Yes | Yes | No | -| GCP account | Connect to Gcp. | Yes | Yes | Yes | -| Login to Docker | Login to docker repository. | Yes | Yes | Yes | -| Deploy Backend | Build docker container, push it to repository, deploy it to Gcp. | Yes | Yes | Yes | - -#### Build And Deploy Playground Frontend Application - [build_playground_frontend.yml](.github/workflows/build_playground_frontend.yml) - -| Job | Description | Pull Request Run | Direct Push/Merge Run | Requires GCP Credentials | -|----------------------------------------|----------------------------------------------------------------------|------------------|-----------------------|--------------------------| -| Check out the repo | GitHub Action used to check-out a repository. | Yes | Yes | No | -| setup-java | Install Java. | Yes | Yes | No | -| install flutter | Install flutter package. | Yes | Yes | No | -| maven config clean | Clean maven settings | Yes | Yes | No | -| install npm | Install nmp package. | Yes | Yes | No | -| lint dockerfile | Install and lint docker file. | Yes | Yes | No | -| setup-terraform | Install terraform. | Yes | Yes | No | -| Docker Tag | Add tag , if it not set. | Yes | Yes | No | -| GCP account | Connect to Gcp. | Yes | Yes | Yes | -| Login to Docker | Login to docker repository. | Yes | Yes | Yes | -| Deploy Frontend | Config and build, push docker container to repository, deploy to Gcp.| Yes | Yes | Yes | - -#### Collect And Deploy Playground Examples - [playground_deploy_examples.yml](.github/workflows/playground_deploy_examples.yml) - -| Job | Description | Pull Request Run | Direct Push/Merge Run | Requires GCP Credentials | -|----------------------------------------|----------------------------------------------------------------------|------------------|-----------------------|--------------------------| -| Check out the repo | GitHub Action used to check-out a repository. | Yes | Yes | No | -| setup-python | Install Python. | Yes | Yes | No | -| setup-java | Install Java. | Yes | Yes | No | -| Install kubectl | Install tool kubectl for kubernetes cloud | Yes | Yes | No | -| Install helm | Install Help plugin | Yes | Yes | No | -| Set up Cloud SDK | Install GCP Cloud client | Yes | Yes | No | -| install deps | Add packages: | Yes | Yes | No | -| | grpcio-tools,grpcio,mock,protobuf, pytest,pytest-mock, | | | | -| | PyYAML,google-cloud-storage,tqdm | | | | -| maven config clean | Clean maven settings | Yes | Yes | No | -| GCP account | Connect to Gcp. | Yes | Yes | Yes | -| Docker Tag | Add tag , if it not set. | Yes | Yes | No | -| Get K8s Config | Get config for kubectl | Yes | Yes | Yes | -| Login to Docker | Login to docker repository. | Yes | Yes | Yes | -| Build And Push Apps | Build and push docker containers for app | Yes | Yes | Yes | -| Install helm chart | Deploy app to kubernetes cloud | Yes | Yes | Yes | -| Run Python Examples CI | Prepare examples for Python | Yes | Yes | Yes | -| Run Python Examples CD | Execute examples for Python | Yes | Yes | Yes | -| Run Go Examples CI | Prepare examples for Go | Yes | Yes | Yes | -| Run Go Examples CD | Execute examples for Go | Yes | Yes | Yes | -| Run Java Examples CI | Prepare examples for Java | Yes | Yes | Yes | -| Run Java Examples CD | Execute examples for Java | Yes | Yes | Yes | -| Delete Helm Chart | Drop kubernetes cloud | Yes | Yes | Yes | - -#### Secrets for action - - - GCP_PLAYGROUND_REGION - gcp main region location cloud (default:us-central1) - - GCP_PLAYGROUND_SA_EMAIL - gcp service account id (default:playground-dep...@apache-beam-testing.iam.gserviceaccount.com) - - GCP_PLAYGROUND_PROJECT_ID - gcp project id (default:apache-beam-testing) - - PLAYGROUND_REGISTRY_NAME - gcp docker registry address (default:us-central1-docker.pkg.dev) - - GCP_PLAYGROUND_SA_KEY - gcp private key file, it export from service account and encode to Base64 - -