This is an automated email from the ASF dual-hosted git repository.
gyfora pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-kubernetes-operator.git
The following commit(s) were added to refs/heads/main by this push:
new 1a5ee081 [FLINK-36460] Refactoring the CI matrix
1a5ee081 is described below
commit 1a5ee081eef2c8435c9414faf2530c2f12149c08
Author: Sam Barker <[email protected]>
AuthorDate: Sat Nov 2 02:05:29 2024 +1300
[FLINK-36460] Refactoring the CI matrix
---
.github/workflows/ci.yml | 235 ++++++++++++++-------------------------------
.github/workflows/e2e.yaml | 96 ++++++++++++++++++
2 files changed, 170 insertions(+), 161 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bff87908..1deb2c2e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -27,10 +27,14 @@ on:
- main
- release-*
pull_request:
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch }}
+ cancel-in-progress: true
+
jobs:
test_ci:
runs-on: ubuntu-latest
- name: test_ci
+ name: maven build
strategy:
matrix:
java-version: [ 11, 17, 21 ]
@@ -68,195 +72,104 @@ jobs:
mvn -B verify -Dit.skip=false
cd ..
e2e_smoke_test:
- name: HTTP Client smoke test
- runs-on: ubuntu-latest
+ name: Smoke test
strategy:
matrix:
http-client: [ "okhttp", "jdk", "jetty", "vertx" ]
- version: ["v1_20"]
- mode: ["native"]
- namespace: ["default"]
- java-version: ["21"]
+ java-version: [ "11", "17", "21" ]
+ uses: ./.github/workflows/e2e.yaml
+ with:
+ java-version: ${{ matrix.java-version }}
+ flink-version: "v1_20"
+ http-client: ${{ matrix.http-client }}
+ test: test_application_operations.sh
+ e2e_namespace_tests:
+ name: Alternative namespace tests
+ needs: e2e_smoke_test
+ strategy:
+ matrix:
+ flink-version:
+ - "v1_20"
+ - "v1_18"
+ - "v1_19"
+ - "v1_17"
+ - "v1_16"
+ mode:
+ - "native"
+ - "standalone"
test:
- test_application_operations.sh
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java-version }}
- uses: actions/setup-java@v4
- with:
- java-version: ${{ matrix.java-version }}
- distribution: 'temurin'
- cache: 'maven'
- - name: Start minikube
- run: |
- source e2e-tests/utils.sh
- start_minikube
- - name: Install cert-manager
- run: |
- kubectl get pods -A
- kubectl apply -f e2e-tests/cert-manager.yaml
- kubectl -n cert-manager wait --all=true --for=condition=Available
--timeout=300s deploy
- - name: Build image
- run: |
- export SHELL=/bin/bash
- export DOCKER_BUILDKIT=1
- eval $(minikube -p minikube docker-env)
- JAVA_VERSION=${{matrix.java-version}}
- HTTP_CLIENT=${{ matrix.http-client }}
- docker build --progress=plain --no-cache -f ./Dockerfile -t
flink-kubernetes-operator:ci-latest --progress plain --build-arg
JAVA_VERSION="${JAVA_VERSION:-11}" --build-arg
HTTP_CLIENT="${HTTP_CLIENT:-okhttp}" .
- docker images
- - name: Start the operator
- run: |
- if [[ "${{ matrix.test }}" == "test_flink_operator_ha.sh" ]]; then
- sed -i "s/# kubernetes.operator.leader-election.enabled:
false/kubernetes.operator.leader-election.enabled: true/"
helm/flink-kubernetes-operator/conf/flink-conf.yaml
- sed -i "s/# kubernetes.operator.leader-election.lease-name:
flink-operator-lease/kubernetes.operator.leader-election.lease-name:
flink-operator-lease/" helm/flink-kubernetes-operator/conf/flink-conf.yaml
- sed -i "s/replicas: 1/replicas: 2/"
helm/flink-kubernetes-operator/values.yaml
- fi
- helm --debug install flink-kubernetes-operator -n ${{
matrix.namespace }} helm/flink-kubernetes-operator --set
image.repository=flink-kubernetes-operator --set image.tag=ci-latest ${{
matrix.extraArgs }}
- kubectl wait --for=condition=Available --timeout=120s -n ${{
matrix.namespace }} deploy/flink-kubernetes-operator
- kubectl get pods -n ${{ matrix.namespace }}
- - name: Run Flink e2e tests
- run: |
- sed -i "s/image: flink:.*/image: ${{ matrix.image }}/"
e2e-tests/data/*.yaml
- sed -i "s/flinkVersion: .*/flinkVersion: ${{ matrix.version }}/"
e2e-tests/data/*.yaml
- sed -i "s/mode: .*/mode: ${{ matrix.mode }}/" e2e-tests/data/*.yaml
- git diff HEAD
- echo "Running e2e-tests/$test"
- bash e2e-tests/${{ matrix.test }} || exit 1
- git reset --hard
- - name: Stop the operator
- run: |
- helm uninstall -n ${{ matrix.namespace }} flink-kubernetes-operator
- - name: Stop minikube
- run: |
- source e2e-tests/utils.sh
- stop_minikube
+ - test_multi_sessionjob.sh
+ - test_autoscaler.sh
+ - test_dynamic_config.sh
+ exclude:
+ - mode: standalone
+ test: test_autoscaler.sh
+ - mode: standalone
+ test: test_dynamic_config.sh
+ - flink-version: v1_16
+ test: test_autoscaler.sh
+ - flink-version: v1_16
+ test: test_dynamic_config.sh
+
+ uses: ./.github/workflows/e2e.yaml
+ with:
+ java-version: 11
+ flink-version: ${{ matrix.flink-version }}
+ test: ${{ matrix.test }}
+ namespace: "flink"
+ create-namespace: true
+ mode: ${{ matrix.mode }}
e2e_ci:
needs: e2e_smoke_test
- runs-on: ubuntu-latest
strategy:
matrix:
- version: ["v1_20","v1_19","v1_18","v1_17","v1_16"]
- namespace: ["default","flink"]
- mode: ["native", "standalone"]
- java-version: [ 11, 17, 21 ]
+ flink-version:
+ - "v1_20"
+ - "v1_18"
+ - "v1_19"
+ - "v1_17"
+ - "v1_16"
+ mode:
+ - "native"
+ - "standalone"
test:
- test_application_kubernetes_ha.sh
- test_application_operations.sh
- test_dynamic_config.sh
- test_sessionjob_kubernetes_ha.sh
- test_sessionjob_operations.sh
- - test_multi_sessionjob.sh
- test_autoscaler.sh
- test_flink_operator_ha.sh
- test_snapshot.sh
- include:
- - namespace: flink
- extraArgs: '--create-namespace --set
"watchNamespaces={default,flink}"'
- - version: v1_20
- image: flink:1.20
- - version: v1_19
- image: flink:1.19
- - version: v1_18
- image: flink:1.18
- - version: v1_17
- image: flink:1.17
- - version: v1_16
- image: flink:1.16
exclude:
- - namespace: default
- test: test_multi_sessionjob.sh
- - namespace: default
- test: test_autoscaler.sh
- - namespace: default
- test: test_dynamic_config.sh
- - mode: standalone
+ - flink-version: v1_16
test: test_autoscaler.sh
- mode: standalone
- test: test_dynamic_config.sh
- - mode: standalone
- test: test_snapshot.sh
- - version: v1_16
test: test_autoscaler.sh
- - version: v1_16
+ - flink-version: v1_16
test: test_dynamic_config.sh
- - version: v1_16
+ - flink-version: v1_16
test: test_flink_operator_ha.sh
- - version: v1_16
+ - flink-version: v1_16
test: test_snapshot.sh
- - version: v1_17
+ - flink-version: v1_17
test: test_dynamic_config.sh
- - version: v1_17
+ - flink-version: v1_17
test: test_flink_operator_ha.sh
- - version: v1_17
+ - flink-version: v1_17
test: test_snapshot.sh
- - version: v1_18
+ - flink-version: v1_18
test: test_dynamic_config.sh
- - version: v1_18
+ - flink-version: v1_18
test: test_flink_operator_ha.sh
- - version: v1_18
+ - flink-version: v1_18
test: test_snapshot.sh
- - version: v1_19
+ - flink-version: v1_19
test: test_snapshot.sh
- - version: v1_16
- java-version: 17
- - version: v1_17
- java-version: 17
- - version: v1_16
- java-version: 21
- - version: v1_17
- java-version: 21
- - version: v1_18
- java-version: 21
- name: e2e_ci
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java-version }}
- uses: actions/setup-java@v4
- with:
- java-version: ${{ matrix.java-version }}
- distribution: 'temurin'
- cache: 'maven'
- - name: Start minikube
- run: |
- source e2e-tests/utils.sh
- start_minikube
- - name: Install cert-manager
- run: |
- kubectl get pods -A
- kubectl apply -f e2e-tests/cert-manager.yaml
- kubectl -n cert-manager wait --all=true --for=condition=Available
--timeout=300s deploy
- - name: Build image
- run: |
- export SHELL=/bin/bash
- export DOCKER_BUILDKIT=1
- eval $(minikube -p minikube docker-env)
- JAVA_VERSION=${{ matrix.java-version }}
- docker build --progress=plain --no-cache -f ./Dockerfile -t
flink-kubernetes-operator:ci-latest --progress plain --build-arg
JAVA_VERSION="${JAVA_VERSION:-11}" .
- docker images
- - name: Start the operator
- run: |
- if [[ "${{ matrix.test }}" == "test_flink_operator_ha.sh" ]]; then
- sed -i "s/# kubernetes.operator.leader-election.enabled:
false/kubernetes.operator.leader-election.enabled: true/"
helm/flink-kubernetes-operator/conf/flink-conf.yaml
- sed -i "s/# kubernetes.operator.leader-election.lease-name:
flink-operator-lease/kubernetes.operator.leader-election.lease-name:
flink-operator-lease/" helm/flink-kubernetes-operator/conf/flink-conf.yaml
- sed -i "s/replicas: 1/replicas: 2/"
helm/flink-kubernetes-operator/values.yaml
- fi
- helm --debug install flink-kubernetes-operator -n ${{
matrix.namespace }} helm/flink-kubernetes-operator --set
image.repository=flink-kubernetes-operator --set image.tag=ci-latest ${{
matrix.extraArgs }}
- kubectl wait --for=condition=Available --timeout=120s -n ${{
matrix.namespace }} deploy/flink-kubernetes-operator
- kubectl get pods -n ${{ matrix.namespace }}
- - name: Run Flink e2e tests
- run: |
- sed -i "s/image: flink:.*/image: ${{ matrix.image }}/"
e2e-tests/data/*.yaml
- sed -i "s/flinkVersion: .*/flinkVersion: ${{ matrix.version }}/"
e2e-tests/data/*.yaml
- sed -i "s/mode: .*/mode: ${{ matrix.mode }}/" e2e-tests/data/*.yaml
- git diff HEAD
- echo "Running e2e-tests/$test"
- bash e2e-tests/${{ matrix.test }} || exit 1
- git reset --hard
- - name: Stop the operator
- run: |
- helm uninstall -n ${{ matrix.namespace }} flink-kubernetes-operator
- - name: Stop minikube
- run: |
- source e2e-tests/utils.sh
- stop_minikube
+ uses: ./.github/workflows/e2e.yaml
+ with:
+ java-version: 11
+ flink-version: ${{ matrix.flink-version }}
+ test: ${{ matrix.test }}
+ mode: ${{ matrix.mode }}
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
new file mode 100644
index 00000000..d349a434
--- /dev/null
+++ b/.github/workflows/e2e.yaml
@@ -0,0 +1,96 @@
+name: E2E test workflow
+
+on:
+ workflow_call:
+ inputs:
+ java-version:
+ required: true
+ type: string
+ flink-version:
+ required: true
+ type: string
+ http-client:
+ required: false
+ type: string
+ default: "okhttp"
+ namespace:
+ required: false
+ type: string
+ default: "default"
+ mode:
+ required: false
+ type: string
+ default: "native"
+ test:
+ required: true
+ type: string
+ create-namespace:
+ type: boolean
+ default: false
+
+jobs:
+ e2e_test:
+ name: ${{ inputs.test }}
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up JDK ${{ inputs.java-version }}
+ uses: actions/setup-java@v4
+ with:
+ java-version: ${{ inputs.java-version }}
+ distribution: 'temurin'
+ cache: 'maven'
+ - name: Start minikube
+ run: |
+ source e2e-tests/utils.sh
+ start_minikube
+ - name: Install cert-manager
+ run: |
+ kubectl get pods -A
+ kubectl apply -f e2e-tests/cert-manager.yaml
+ kubectl -n cert-manager wait --all=true --for=condition=Available
--timeout=300s deploy
+ - name: Build image
+ run: |
+ export SHELL=/bin/bash
+ export DOCKER_BUILDKIT=1
+ eval $(minikube -p minikube docker-env)
+ JAVA_VERSION=${{ inputs.java-version }}
+ HTTP_CLIENT=${{ inputs.http-client }}
+ docker build --progress=plain --no-cache -f ./Dockerfile -t
flink-kubernetes-operator:ci-latest --progress plain --build-arg
JAVA_VERSION="${JAVA_VERSION:-11}" --build-arg
HTTP_CLIENT="${HTTP_CLIENT:-okhttp}" .
+ docker images
+ - name: Create ${{ inputs.namespace }} == ${{ inputs.create-namespace }}
+ id: namespace-creator
+ if: ${{ inputs.create-namespace }}
+ run: |
+ source e2e-tests/utils.sh
+ create_namespace ${{ inputs.namespace }}
+ echo 'EXTRA_HELM_INSTALL_ARGS=--set
"watchNamespaces={default,flink}"' >> $GITHUB_OUTPUT
+ - name: Start the operator
+ run: |
+ if [[ "${{ inputs.test }}" == "test_flink_operator_ha.sh" ]]; then
+ sed -i "s/# kubernetes.operator.leader-election.enabled:
false/kubernetes.operator.leader-election.enabled: true/"
helm/flink-kubernetes-operator/conf/flink-conf.yaml
+ sed -i "s/# kubernetes.operator.leader-election.lease-name:
flink-operator-lease/kubernetes.operator.leader-election.lease-name:
flink-operator-lease/" helm/flink-kubernetes-operator/conf/flink-conf.yaml
+ sed -i "s/replicas: 1/replicas: 2/"
helm/flink-kubernetes-operator/values.yaml
+ fi
+ helm --debug install flink-kubernetes-operator -n ${{
inputs.namespace }} helm/flink-kubernetes-operator --set
image.repository=flink-kubernetes-operator --set image.tag=ci-latest ${{
steps.namespace-creator.outputs.EXTRA_HELM_INSTALL_ARGS }}
+ kubectl wait --for=condition=Available --timeout=120s -n ${{
inputs.namespace }} deploy/flink-kubernetes-operator
+ kubectl get pods -n ${{ inputs.namespace }}
+ - name: Run Flink e2e tests
+ run: |
+ FLINK_IMAGE=$(sed --regexp-extended
's/v([0-9]+)_([0-9]+)/flink:\1.\2/g' <<< ${{ inputs.flink-version }} )
+ echo FLINK_IMAGE=${FLINK_IMAGE}
+ sed -i "s/image: flink:.*/image: ${FLINK_IMAGE}/"
e2e-tests/data/*.yaml
+ sed -i "s/flinkVersion: .*/flinkVersion: ${{ inputs.flink-version
}}/" e2e-tests/data/*.yaml
+ sed -i "s/mode: .*/mode: ${{ inputs.mode }}/" e2e-tests/data/*.yaml
+ git diff HEAD
+ echo "Running e2e-tests/$test"
+ bash e2e-tests/${{ inputs.test }} || exit 1
+ git reset --hard
+ - name: Stop the operator
+ run: |
+ helm uninstall -n ${{ inputs.namespace }} flink-kubernetes-operator
+ - name: Stop minikube
+ run: |
+ source e2e-tests/utils.sh
+ stop_minikube
+
