This is an automated email from the ASF dual-hosted git repository. martijnvisser pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/flink-connector-aws.git
commit d1a3f638754666019a59912cff28fb4b46445664 Author: Martijn Visser <mvis...@confluent.io> AuthorDate: Thu Jun 15 10:32:37 2023 +0200 Revert "[hotfix] Sync CI workflow against other ASF Flink externalized connectors" This reverts commit 31eda54b6a2c64823b1adf7d4b7f943757056924. --- .github/workflows/common.yml | 120 ++++++++++++++++++++++++++++++++++++++++++ .github/workflows/nightly.yml | 6 +-- .github/workflows/push_pr.yml | 8 +-- 3 files changed, 128 insertions(+), 6 deletions(-) diff --git a/.github/workflows/common.yml b/.github/workflows/common.yml new file mode 100644 index 0000000..237a779 --- /dev/null +++ b/.github/workflows/common.yml @@ -0,0 +1,120 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +on: + workflow_call: + inputs: + flink_url: + description: "Url to Flink binary." + required: true + type: string + flink_version: + description: "Flink version to test against." + required: true + type: string + cache_flink_binary: + description: "Whether to cache the Flink binary. Should be false for SNAPSHOT URLs, true otherwise." + required: true + type: boolean + timeout_global: + description: "The timeout in minutes for the entire workflow." + required: false + type: number + default: 60 + timeout_test: + description: "The timeout in minutes for the compile and test step." + required: false + type: number + default: 50 + +jobs: + compile_and_test: + runs-on: ubuntu-latest + strategy: + matrix: + jdk: [ 8, 11 ] + timeout-minutes: ${{ inputs.timeout_global }} + env: + MVN_COMMON_OPTIONS: -U -B --no-transfer-progress -Dflink.version=${{ inputs.flink_version }} + MVN_CONNECTION_OPTIONS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + FLINK_CACHE_DIR: "/tmp/cache/flink" + MVN_BUILD_OUTPUT_FILE: "/tmp/mvn_build_output.out" + MVN_VALIDATION_DIR: "/tmp/flink-validation-deployment" + steps: + - run: echo "Running CI pipeline for JDK version ${{ matrix.jdk }}" + + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Set JDK + uses: actions/setup-java@v3 + with: + java-version: ${{ matrix.jdk }} + distribution: 'temurin' + cache: 'maven' + + - name: Set Maven 3.8.5 + uses: stCarolas/setup-maven@v4.5 + with: + maven-version: 3.8.5 + + - name: Create cache dirs + run: mkdir -p ${{ env.FLINK_CACHE_DIR }} + + - name: Cache Flink binary + if: ${{ inputs.cache_flink_binary }} + uses: actions/cache@v3 + id: cache-flink + with: + path: ${{ env.FLINK_CACHE_DIR }} + key: ${{ inputs.flink_url }} + + - name: Download Flink binary + working-directory: ${{ env.FLINK_CACHE_DIR }} + if: steps.cache-flink.outputs.cache-hit != 'true' + run: wget -q -c ${{ inputs.flink_url }} -O - | tar -xz + + - name: Compile and test flink-connector-aws + timeout-minutes: ${{ inputs.timeout_test }} + run: | + set -o pipefail + + mvn clean install -Dflink.convergence.phase=install -Pcheck-convergence -U -B ${{ env.MVN_CONNECTION_OPTIONS }} \ + -DaltDeploymentRepository=validation_repository::default::file:${{ env.MVN_VALIDATION_DIR }} \ + -Dflink.version=${{ inputs.flink_version }} | tee ${{ env.MVN_BUILD_OUTPUT_FILE }} + + - name: Run e2e tests + run: | + set -o pipefail + + cd flink-connector-aws-e2e-tests + + mvn clean verify ${{ env.MVN_CONNECTION_OPTIONS }} \ + -DaltDeploymentRepository=validation_repository::default::file:${{ env.MVN_VALIDATION_DIR }} \ + -Dflink.version=${{ inputs.flink_version }} \ + -Prun-end-to-end-tests -DdistDir=${{ env.FLINK_CACHE_DIR }}/flink-${{ inputs.flink_version }} \ + | tee ${{ env.MVN_BUILD_OUTPUT_FILE }} + + mvn clean + + - name: Check licensing + run: | + mvn ${MVN_COMMON_OPTIONS} exec:java@check-license -N \ + -Dexec.args="${{ env.MVN_BUILD_OUTPUT_FILE }} $(pwd) $(pwd)" \ + ${{ env.MVN_CONNECTION_OPTIONS }} \ + -Dlog4j.configurationFile=file://$(pwd)/tools/ci/log4j.properties diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index f67c05b..7084a92 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -20,14 +20,14 @@ name: "flink-connector-aws: nightly build" on: schedule: - cron: "0 0 * * *" - workflow_dispatch: jobs: compile_and_test: if: github.repository_owner == 'apache' strategy: matrix: flink: [1.16-SNAPSHOT, 1.17-SNAPSHOT, 1.18-SNAPSHOT] - uses: apache/flink-connector-shared-utils/.github/workflows/ci.yml@ci_utils + uses: ./.github/workflows/common.yml with: flink_version: ${{ matrix.flink }} - run_dependency_convergence: false \ No newline at end of file + flink_url: https://s3.amazonaws.com/flink-nightly/flink-${{ matrix.flink }}-bin-scala_2.12.tgz + cache_flink_binary: false diff --git a/.github/workflows/push_pr.yml b/.github/workflows/push_pr.yml index 2ba2dc9..54bcd88 100644 --- a/.github/workflows/push_pr.yml +++ b/.github/workflows/push_pr.yml @@ -23,9 +23,11 @@ concurrency: cancel-in-progress: true jobs: compile_and_test: + uses: ./.github/workflows/common.yml strategy: matrix: - flink: [1.16-SNAPSHOT, 1.17-SNAPSHOT] - uses: apache/flink-connector-shared-utils/.github/workflows/ci.yml@ci_utils + flink: [1.16.1, 1.17.0] with: - flink_version: ${{ matrix.flink }} \ No newline at end of file + flink_version: ${{ matrix.flink }} + flink_url: https://archive.apache.org/dist/flink/flink-${{ matrix.flink }}/flink-${{ matrix.flink }}-bin-scala_2.12.tgz + cache_flink_binary: true