This is an automated email from the ASF dual-hosted git repository.
zhaijia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new fc5ae1d [Build] Refactor integration test commands to
run_integration_group.sh script (#9770)
fc5ae1d is described below
commit fc5ae1d454dd96ecfbe0d453e1ddf5258097d520
Author: Lari Hotari <lhot...@users.noreply.github.com>
AuthorDate: Tue Mar 2 12:20:29 2021 +0200
[Build] Refactor integration test commands to run_integration_group.sh
script (#9770)
### Motivation
- Add a similar solution for running integration tests as there is for
running unit tests in run_unit_group.sh script
This benefit of this change is that it will be easier to refactor the
GitHub Actions workflow build when the commands to run the integration tests
are maintained in a separate bash script.
### Modifications
- Move commands to run integration tests from the GitHub Actions workflow
yaml files to a bash script `build/run_ingration_group.sh`.
- The commands aren't modified as part of this PR, only moved to the script.
---
.../ci-integration-backwards-compatibility.yaml | 2 +-
.github/workflows/ci-integration-cli.yaml | 8 +-
.../workflows/ci-integration-function-state.yaml | 2 +-
.github/workflows/ci-integration-messaging.yaml | 12 +-
.github/workflows/ci-integration-process.yaml | 12 +-
.github/workflows/ci-integration-schema.yaml | 2 +-
.github/workflows/ci-integration-sql.yaml | 2 +-
.github/workflows/ci-integration-standalone.yaml | 2 +-
.github/workflows/ci-integration-thread.yaml | 10 +-
.../ci-integration-tiered-filesystem.yaml | 2 +-
.../workflows/ci-integration-tiered-jcloud.yaml | 2 +-
.github/workflows/ci-integration-transaction.yaml | 2 +-
.github/workflows/ci-shade-test.yaml | 2 +-
build/run_integration_group.sh | 131 +++++++++++++++++++++
14 files changed, 147 insertions(+), 44 deletions(-)
diff --git a/.github/workflows/ci-integration-backwards-compatibility.yaml
b/.github/workflows/ci-integration-backwards-compatibility.yaml
index 4fc4e1a..f9fe802 100644
--- a/.github/workflows/ci-integration-backwards-compatibility.yaml
+++ b/.github/workflows/ci-integration-backwards-compatibility.yaml
@@ -90,4 +90,4 @@ jobs:
- name: run integration tests
if: steps.docs.outputs.changed_only == 'no'
- run: ./build/retry.sh mvn -B -f tests/pom.xml test
-DintegrationTestSuiteFile=pulsar-backwards-compatibility.xml
-DintegrationTests -DredirectTestOutputToFile=false
+ run: ./build/run_integration_group.sh BACKWARDS_COMPAT
diff --git a/.github/workflows/ci-integration-cli.yaml
b/.github/workflows/ci-integration-cli.yaml
index 7488b55..3500956 100644
--- a/.github/workflows/ci-integration-cli.yaml
+++ b/.github/workflows/ci-integration-cli.yaml
@@ -88,11 +88,7 @@ jobs:
if: steps.docs.outputs.changed_only == 'no'
run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker
-DskipTests
- - name: run pulsar cli integration tests
+ - name: run integration tests
if: steps.docs.outputs.changed_only == 'no'
- run: mvn -B -f tests/pom.xml test
-DintegrationTestSuiteFile=pulsar-cli.xml -DintegrationTests
-DredirectTestOutputToFile=false
-
- - name: run pulsar auth integration tests
- if: steps.docs.outputs.changed_only == 'no'
- run: mvn -B -f tests/pom.xml test
-DintegrationTestSuiteFile=pulsar-auth.xml -DintegrationTests
-DredirectTestOutputToFile=false
+ run: ./build/run_integration_group.sh CLI
diff --git a/.github/workflows/ci-integration-function-state.yaml
b/.github/workflows/ci-integration-function-state.yaml
index 9f47c8b..6595f10 100644
--- a/.github/workflows/ci-integration-function-state.yaml
+++ b/.github/workflows/ci-integration-function-state.yaml
@@ -90,4 +90,4 @@ jobs:
- name: run integration tests
if: steps.docs.outputs.changed_only == 'no'
- run: mvn -B -f tests/pom.xml test
-DintegrationTestSuiteFile=pulsar-function-state.xml -DintegrationTests
-DredirectTestOutputToFile=false
+ run: ./build/run_integration_group.sh FUNCTION_STATE
\ No newline at end of file
diff --git a/.github/workflows/ci-integration-messaging.yaml
b/.github/workflows/ci-integration-messaging.yaml
index 6b13630..1decf8b 100644
--- a/.github/workflows/ci-integration-messaging.yaml
+++ b/.github/workflows/ci-integration-messaging.yaml
@@ -88,14 +88,6 @@ jobs:
if: steps.docs.outputs.changed_only == 'no'
run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker
-DskipTests -Ddocker.nocache=true
- - name: run integration messaging tests
+ - name: run integration tests
if: steps.docs.outputs.changed_only == 'no'
- run: mvn -B -f tests/pom.xml test
-DintegrationTestSuiteFile=pulsar-messaging.xml -DintegrationTests
-DredirectTestOutputToFile=false
-
- - name: run integration proxy tests
- if: steps.docs.outputs.changed_only == 'no'
- run: mvn -B -f tests/pom.xml test
-DintegrationTestSuiteFile=pulsar-proxy.xml -DintegrationTests
-DredirectTestOutputToFile=false
-
- - name: run integration proxy with WebSocket tests
- if: steps.docs.outputs.changed_only == 'no'
- run: mvn -B -f tests/pom.xml test
-DintegrationTestSuiteFile=pulsar-proxy-websocket.xml -DintegrationTests
-DredirectTestOutputToFile=false
+ run: ./build/run_integration_group.sh MESSAGING
\ No newline at end of file
diff --git a/.github/workflows/ci-integration-process.yaml
b/.github/workflows/ci-integration-process.yaml
index 1a1fdfa..25b023b 100644
--- a/.github/workflows/ci-integration-process.yaml
+++ b/.github/workflows/ci-integration-process.yaml
@@ -87,14 +87,6 @@ jobs:
if: steps.docs.outputs.changed_only == 'no'
run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker
-DskipTests
- - name: run integration function
+ - name: run integration tests
if: steps.docs.outputs.changed_only == 'no'
- run: ./build/retry.sh mvn -B -f tests/pom.xml test
-DintegrationTestSuiteFile=pulsar-process.xml -DintegrationTests
-DredirectTestOutputToFile=false -Dgroups=function
-
- - name: run integration source
- if: steps.docs.outputs.changed_only == 'no'
- run: ./build/retry.sh mvn -B -f tests/pom.xml test
-DintegrationTestSuiteFile=pulsar-process.xml -DintegrationTests
-DredirectTestOutputToFile=false -Dgroups=source
-
- - name: run integraion sink
- if: steps.docs.outputs.changed_only == 'no'
- run: ./build/retry.sh mvn -B -f tests/pom.xml test
-DintegrationTestSuiteFile=pulsar-process.xml -DintegrationTests
-DredirectTestOutputToFile=false -Dgroups=sink
+ run: ./build/run_integration_group.sh PULSAR_CONNECTORS_PROCESS
\ No newline at end of file
diff --git a/.github/workflows/ci-integration-schema.yaml
b/.github/workflows/ci-integration-schema.yaml
index f4341af..85d099b 100644
--- a/.github/workflows/ci-integration-schema.yaml
+++ b/.github/workflows/ci-integration-schema.yaml
@@ -86,4 +86,4 @@ jobs:
- name: run integration tests
if: steps.docs.outputs.changed_only == 'no'
- run: ./build/retry.sh mvn -B -f tests/pom.xml test
-DintegrationTestSuiteFile=pulsar-schema.xml -DintegrationTests
-DredirectTestOutputToFile=false
+ run: ./build/run_integration_group.sh SCHEMA
\ No newline at end of file
diff --git a/.github/workflows/ci-integration-sql.yaml
b/.github/workflows/ci-integration-sql.yaml
index bc87a23..159b70d 100644
--- a/.github/workflows/ci-integration-sql.yaml
+++ b/.github/workflows/ci-integration-sql.yaml
@@ -89,4 +89,4 @@ jobs:
- name: run integration tests
if: steps.docs.outputs.changed_only == 'no'
- run: mvn -B -f tests/pom.xml test
-DintegrationTestSuiteFile=pulsar-sql.xml -DintegrationTests
-DredirectTestOutputToFile=false -DtestForkCount=1
+ run: ./build/run_integration_group.sh SQL
diff --git a/.github/workflows/ci-integration-standalone.yaml
b/.github/workflows/ci-integration-standalone.yaml
index 56189c4..5d6c0e1 100644
--- a/.github/workflows/ci-integration-standalone.yaml
+++ b/.github/workflows/ci-integration-standalone.yaml
@@ -89,4 +89,4 @@ jobs:
- name: run integration tests
if: steps.docs.outputs.changed_only == 'no'
- run: mvn -B -f tests/pom.xml test
-DintegrationTestSuiteFile=pulsar-standalone.xml -DintegrationTests
-DredirectTestOutputToFile=false
+ run: ./build/run_integration_group.sh STANDALONE
diff --git a/.github/workflows/ci-integration-thread.yaml
b/.github/workflows/ci-integration-thread.yaml
index 24700ec..70c4823 100644
--- a/.github/workflows/ci-integration-thread.yaml
+++ b/.github/workflows/ci-integration-thread.yaml
@@ -89,12 +89,4 @@ jobs:
- name: run integration function
if: steps.docs.outputs.changed_only == 'no'
- run: ./build/retry.sh mvn -B -f tests/pom.xml test
-DintegrationTestSuiteFile=pulsar-thread.xml -DintegrationTests
-DredirectTestOutputToFile=false -Dgroups=function
-
- - name: run integration source
- if: steps.docs.outputs.changed_only == 'no'
- run: ./build/retry.sh mvn -B -f tests/pom.xml test
-DintegrationTestSuiteFile=pulsar-thread.xml -DintegrationTests
-DredirectTestOutputToFile=false -Dgroups=source
-
- - name: run integraion sink
- if: steps.docs.outputs.changed_only == 'no'
- run: ./build/retry.sh mvn -B -f tests/pom.xml test
-DintegrationTestSuiteFile=pulsar-thread.xml -DintegrationTests
-DredirectTestOutputToFile=false -Dgroups=sink
+ run: ./build/run_integration_group.sh PULSAR_CONNECTORS_THREAD
\ No newline at end of file
diff --git a/.github/workflows/ci-integration-tiered-filesystem.yaml
b/.github/workflows/ci-integration-tiered-filesystem.yaml
index 38e4f9a..6c3a6c1 100644
--- a/.github/workflows/ci-integration-tiered-filesystem.yaml
+++ b/.github/workflows/ci-integration-tiered-filesystem.yaml
@@ -89,4 +89,4 @@ jobs:
- name: run integration tests
if: steps.docs.outputs.changed_only == 'no'
- run: ./build/retry.sh mvn -B -f tests/pom.xml test
-DintegrationTestSuiteFile=tiered-filesystem-storage.xml -DintegrationTests
-DredirectTestOutputToFile=false
+ run: ./build/run_integration_group.sh TIERED_FILESYSTEM
diff --git a/.github/workflows/ci-integration-tiered-jcloud.yaml
b/.github/workflows/ci-integration-tiered-jcloud.yaml
index 8c2a6e2..91f9e1b 100644
--- a/.github/workflows/ci-integration-tiered-jcloud.yaml
+++ b/.github/workflows/ci-integration-tiered-jcloud.yaml
@@ -89,4 +89,4 @@ jobs:
- name: run integration tests
if: steps.docs.outputs.changed_only == 'no'
- run: mvn -B -f tests/pom.xml test
-DintegrationTestSuiteFile=tiered-jcloud-storage.xml -DintegrationTests
-DredirectTestOutputToFile=false
+ run: ./build/run_integration_group.sh TIERED_JCLOUD
diff --git a/.github/workflows/ci-integration-transaction.yaml
b/.github/workflows/ci-integration-transaction.yaml
index 07f20c7..10df2fb 100644
--- a/.github/workflows/ci-integration-transaction.yaml
+++ b/.github/workflows/ci-integration-transaction.yaml
@@ -86,4 +86,4 @@ jobs:
- name: run integration tests
if: steps.docs.outputs.changed_only == 'no'
- run: ./build/retry.sh mvn -B -f tests/pom.xml test
-DintegrationTestSuiteFile=pulsar-transaction.xml -DintegrationTests
-DredirectTestOutputToFile=false
+ run: ./build/run_integration_group.sh TRANSACTION
diff --git a/.github/workflows/ci-shade-test.yaml
b/.github/workflows/ci-shade-test.yaml
index 9aa8836..3d5d69f 100644
--- a/.github/workflows/ci-shade-test.yaml
+++ b/.github/workflows/ci-shade-test.yaml
@@ -90,4 +90,4 @@ jobs:
- name: run shade tests
if: steps.docs.outputs.changed_only == 'no'
- run: mvn -B -f tests/pom.xml test -DShadeTests
-DredirectTestOutputToFile=false
+ run: ./build/run_integration_group.sh SHADE
diff --git a/build/run_integration_group.sh b/build/run_integration_group.sh
new file mode 100755
index 0000000..1b659c9
--- /dev/null
+++ b/build/run_integration_group.sh
@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+set -e
+set -o pipefail
+set -o errexit
+
+TEST_GROUP=$1
+if [ -z "$TEST_GROUP" ]; then
+ echo "usage: $0 [test_group]"
+ exit 1
+fi
+shift
+
+# runs integration tests
+mvn_run_integration_test() {
+ (
+ RETRY=""
+ # wrap with retry.sh script if next parameter is "--retry"
+ if [[ "$1" == "--retry" ]]; then
+ RETRY="./build/retry.sh"
+ shift
+ fi
+ # skip wrapping with retry.sh script if next parameter is "--no-retry"
+ if [[ "$1" == "--no-retry" ]]; then
+ RETRY=""
+ shift
+ fi
+ set -x
+ # run the integration tests
+ $RETRY mvn -B -ntp -DredirectTestOutputToFile=false -f tests/pom.xml test
"$@"
+ )
+}
+
+test_group_shade() {
+ mvn_run_integration_test "$@" -DShadeTests
+}
+
+test_group_backwards_compat() {
+ mvn_run_integration_test --retry "$@"
-DintegrationTestSuiteFile=pulsar-backwards-compatibility.xml -DintegrationTests
+}
+
+test_group_cli() {
+ # run pulsar cli integration tests
+ mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-cli.xml
-DintegrationTests
+ # run pulsar auth integration tests
+ mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-auth.xml
-DintegrationTests
+}
+
+test_group_function_state() {
+ mvn_run_integration_test "$@"
-DintegrationTestSuiteFile=pulsar-function-state.xml -DintegrationTests
+}
+
+test_group_messaging() {
+ # run integration messaging tests
+ mvn_run_integration_test "$@"
-DintegrationTestSuiteFile=pulsar-messaging.xml -DintegrationTests
+ # run integration proxy tests
+ mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-proxy.xml
-DintegrationTests
+ # run integration proxy with WebSocket tests
+ mvn_run_integration_test "$@"
-DintegrationTestSuiteFile=pulsar-proxy-websocket.xml -DintegrationTests
+}
+
+test_group_schema() {
+ mvn_run_integration_test --retry "$@"
-DintegrationTestSuiteFile=pulsar-schema.xml -DintegrationTests
+}
+
+test_group_standalone() {
+ mvn_run_integration_test "$@"
-DintegrationTestSuiteFile=pulsar-standalone.xml -DintegrationTests
+}
+
+test_group_transaction() {
+ mvn_run_integration_test --retry "$@"
-DintegrationTestSuiteFile=pulsar-transaction.xml -DintegrationTests
+}
+
+test_group_tiered_filesystem() {
+ mvn_run_integration_test --retry "$@"
-DintegrationTestSuiteFile=tiered-filesystem-storage.xml -DintegrationTests
+}
+
+test_group_tiered_jcloud() {
+ mvn_run_integration_test "$@"
-DintegrationTestSuiteFile=tiered-jcloud-storage.xml -DintegrationTests
+}
+
+test_group_pulsar_connectors_thread() {
+ # run integration function
+ mvn_run_integration_test --retry "$@"
-DintegrationTestSuiteFile=pulsar-thread.xml -DintegrationTests
-Dgroups=function
+ # run integration source
+ mvn_run_integration_test --retry "$@"
-DintegrationTestSuiteFile=pulsar-thread.xml -DintegrationTests -Dgroups=source
+ # run integration sink
+ mvn_run_integration_test --retry "$@"
-DintegrationTestSuiteFile=pulsar-thread.xml -DintegrationTests -Dgroups=sink
+}
+
+test_group_pulsar_connectors_process() {
+ # run integration function
+ mvn_run_integration_test --retry "$@"
-DintegrationTestSuiteFile=pulsar-process.xml -DintegrationTests
-Dgroups=function
+ # run integration source
+ mvn_run_integration_test --retry "$@"
-DintegrationTestSuiteFile=pulsar-process.xml -DintegrationTests -Dgroups=source
+ # run integraion sink
+ mvn_run_integration_test --retry "$@"
-DintegrationTestSuiteFile=pulsar-process.xml -DintegrationTests -Dgroups=sink
+}
+
+test_group_sql() {
+ mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-sql.xml
-DintegrationTests -DtestForkCount=1
+}
+
+echo "Test Group : $TEST_GROUP"
+test_group_function_name="test_group_$(echo "$TEST_GROUP" | tr '[:upper:]'
'[:lower:]')"
+if [[ "$(LC_ALL=C type -t $test_group_function_name)" == "function" ]]; then
+ eval "$test_group_function_name" "$@"
+else
+ echo "INVALID TEST GROUP"
+ exit 1
+fi
\ No newline at end of file
