This is an automated email from the ASF dual-hosted git repository. yux pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink-cdc.git
commit 35df7906400aca08d53e6ae1a85a649f53f2db81 Author: yuxiqian <[email protected]> AuthorDate: Mon Jan 5 19:23:14 2026 +0800 [ci] Combine test modules together to reduce test matrix size ...as we may need more Flink versions to test against later. --- .github/workflows/flink_cdc_base.yml | 196 +---------------------------- .github/workflows/flink_cdc_ci.yml | 4 +- .github/workflows/flink_cdc_ci_nightly.yml | 4 +- .github/workflows/modules.py | 182 +++++++++++++++++++++++++++ 4 files changed, 189 insertions(+), 197 deletions(-) diff --git a/.github/workflows/flink_cdc_base.yml b/.github/workflows/flink_cdc_base.yml index 3cbcd8ac5..7a2c7b95a 100644 --- a/.github/workflows/flink_cdc_base.yml +++ b/.github/workflows/flink_cdc_base.yml @@ -42,109 +42,6 @@ on: required: false type: string -env: - MODULES_CORE: "\ - flink-cdc-cli,\ - flink-cdc-common,\ - flink-cdc-composer,\ - flink-cdc-runtime,\ - flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base,\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values" - - MODULES_PIPELINE_CONNECTORS: "\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris,\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch,\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg,\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka,\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute,\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql,\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase,\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute,\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-postgres,\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon,\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks,\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-fluss,\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-hudi,\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values" - - MODULES_MYSQL_SOURCE: "\ - flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc,\ - flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc" - - MODULES_MYSQL_PIPELINE: "\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql" - - MODULES_POSTGRES_SOURCE: "\ - flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc,\ - flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-postgres-cdc" - - MODULES_POSTGRES_PIPELINE: "\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-postgres" - - MODULES_ORACLE: "\ - flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc,\ - flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oracle-cdc" - - MODULES_MONGODB: "\ - flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc,\ - flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mongodb-cdc" - - MODULES_SQLSERVER: "\ - flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc,\ - flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-sqlserver-cdc" - - MODULES_TIDB: "\ - flink-cdc-connect/flink-cdc-source-connectors/flink-connector-tidb-cdc,\ - flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-tidb-cdc" - - MODULES_OCEANBASE_SOURCE: "\ - flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc,\ - flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc" - - MODULES_OCEANBASE_PIPELINE: "\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase" - - MODULES_DB2: "\ - flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc,\ - flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-db2-cdc" - - MODULES_VITESS: "\ - flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc,\ - flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-vitess-cdc" - - MODULES_DORIS: "\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris" - - MODULES_STARROCKS: "\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks" - - MODULES_ICEBERG: "\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg" - - MODULES_KAFKA: "\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka" - - MODULES_PAIMON: "\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon" - - MODULES_ELASTICSEARCH: "\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch" - - MODULES_MAXCOMPUTE: "\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute" - - MODULES_FLUSS: "\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-fluss" - - MODULES_HUDI: "\ - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-hudi" - - MODULES_PIPELINE_E2E: "\ - flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests" - - MODULES_SOURCE_E2E: "\ - flink-cdc-e2e-tests/flink-cdc-source-e2e-tests" - jobs: test: runs-on: ubuntu-latest @@ -199,98 +96,11 @@ jobs: echo "JVM timezone is set to $jvm_timezone" set -o pipefail - case ${{ matrix.module }} in - ("core") - modules=${{ env.MODULES_CORE }} - ;; - ("mysql-source") - modules=${{ env.MODULES_MYSQL_SOURCE }} - ;; - ("mysql-pipeline") - modules=${{ env.MODULES_MYSQL_PIPELINE }} - ;; - ("postgres-source") - modules=${{ env.MODULES_POSTGRES_SOURCE }} - ;; - ("postgres-pipeline") - modules=${{ env.MODULES_POSTGRES_PIPELINE }} - ;; - ("oracle") - modules=${{ env.MODULES_ORACLE }} - ;; - ("mongodb7") - modules=${{ env.MODULES_MONGODB }} - ;; - ("mongodb8") - modules=${{ env.MODULES_MONGODB }} - ;; - ("sqlserver") - modules=${{ env.MODULES_SQLSERVER }} - ;; - ("tidb") - modules=${{ env.MODULES_TIDB }} - ;; - ("oceanbase-source") - modules=${{ env.MODULES_OCEANBASE_SOURCE }} - ;; - ("oceanbase-pipeline") - modules=${{ env.MODULES_OCEANBASE_PIPELINE }} - ;; - ("db2") - modules=${{ env.MODULES_DB2 }} - ;; - ("vitess") - modules=${{ env.MODULES_VITESS }} - ;; - ("doris") - modules=${{ env.MODULES_DORIS }} - ;; - ("elasticsearch") - modules=${{ env.MODULES_ELASTICSEARCH }} - ;; - ("iceberg") - modules=${{ env.MODULES_ICEBERG }} - ;; - ("kafka") - modules=${{ env.MODULES_KAFKA }} - ;; - ("maxcompute") - modules=${{ env.MODULES_MAXCOMPUTE }} - ;; - ("paimon") - modules=${{ env.MODULES_PAIMON }} - ;; - ("starrocks") - modules=${{ env.MODULES_STARROCKS }} - ;; - ("fluss") - modules=${{ env.MODULES_FLUSS }} - ;; - ("hudi") - modules=${{ env.MODULES_HUDI }} - ;; - ("pipeline_e2e") - compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL_SOURCE }},${{ env.MODULES_POSTGRES_SOURCE }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE_SOURCE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_PIPELINE_E2E }}" - modules=${{ env.MODULES_PIPELINE_E2E }} - ;; - ("source_e2e") - compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL_SOURCE }},${{ env.MODULES_POSTGRES_SOURCE }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE_SOURCE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_SOURCE_E2E }}" - modules=${{ env.MODULES_SOURCE_E2E }} - ;; - esac + modules=$(./.github/workflows/modules.py test "${{ matrix.module }}") + compile_modules=$(./.github/workflows/modules.py compile "${{ matrix.module }}") - if [ ${{ matrix.module }} != "pipeline_e2e" ] && [ ${{ matrix.module }} != "source_e2e" ]; then - compile_modules=$modules - fi - - build_maven_parameter="" + build_maven_parameter="-DspecifiedMongoVersion=8.0.14" - if [ ${{ matrix.module }} == "mongodb7" ]; then - build_maven_parameter="-DspecifiedMongoVersion=7.0.24" - elif [ ${{ matrix.module }} == "mongodb8" ]; then - build_maven_parameter="-DspecifiedMongoVersion=8.0.14" - fi - if [ ! -z "${{ matrix.flink-version }}" ]; then build_maven_parameter="${build_maven_parameter:+$build_maven_parameter }-DspecifiedFlinkVersion=${{ matrix.flink-version }}" fi diff --git a/.github/workflows/flink_cdc_ci.yml b/.github/workflows/flink_cdc_ci.yml index 92f6a3e4f..cf0070768 100644 --- a/.github/workflows/flink_cdc_ci.yml +++ b/.github/workflows/flink_cdc_ci.yml @@ -68,13 +68,13 @@ jobs: uses: ./.github/workflows/flink_cdc_base.yml with: java-versions: "[11]" - modules: "['mysql-pipeline', 'postgres-pipeline', 'oceanbase-pipeline', 'doris', 'elasticsearch', 'iceberg', 'kafka', 'maxcompute', 'paimon', 'starrocks', 'fluss', 'hudi']" + modules: "['doris, elasticsearch, fluss, hudi, iceberg, kafka, maxcompute, mysql-pipeline, oceanbase-pipeline, paimon, postgres-pipeline, starrocks']" source-ut: name: Source Unit Tests uses: ./.github/workflows/flink_cdc_base.yml with: java-versions: "[11]" - modules: "['mysql-source', 'postgres-source', 'oracle', 'mongodb7', 'mongodb8', 'sqlserver', 'tidb', 'oceanbase-source', 'db2', 'vitess']" + modules: "['mysql-source', 'postgres-source, oceanbase-source, tidb, vitess', 'oracle, sqlserver', 'db2, mongodb']" pipeline_e2e: strategy: fail-fast: false diff --git a/.github/workflows/flink_cdc_ci_nightly.yml b/.github/workflows/flink_cdc_ci_nightly.yml index 3a3969543..8a0ce1ff4 100644 --- a/.github/workflows/flink_cdc_ci_nightly.yml +++ b/.github/workflows/flink_cdc_ci_nightly.yml @@ -60,14 +60,14 @@ jobs: uses: ./.github/workflows/flink_cdc_base.yml with: java-versions: "[17]" - modules: "['mysql-pipeline', 'postgres-pipeline', 'oceanbase-pipeline', 'doris', 'elasticsearch', 'iceberg', 'kafka', 'maxcompute', 'paimon', 'starrocks', 'fluss', 'hudi']" + modules: "['doris, elasticsearch, fluss, hudi, iceberg, kafka, maxcompute, mysql-pipeline, oceanbase-pipeline, paimon, postgres-pipeline, starrocks']" source-ut: if: github.repository == 'apache/flink-cdc' name: Source Unit Tests uses: ./.github/workflows/flink_cdc_base.yml with: java-versions: "[17]" - modules: "['mysql-source', 'postgres-source', 'oracle', 'mongodb7', 'mongodb8', 'sqlserver', 'tidb', 'oceanbase-source', 'db2', 'vitess']" + modules: "['mysql-source', 'postgres-source, oceanbase-source, tidb, vitess', 'oracle, sqlserver', 'db2, mongodb']" pipeline_e2e: if: github.repository == 'apache/flink-cdc' strategy: diff --git a/.github/workflows/modules.py b/.github/workflows/modules.py new file mode 100755 index 000000000..c4e2c2c78 --- /dev/null +++ b/.github/workflows/modules.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 + +import sys + +if len(sys.argv) != 3: + raise RuntimeError(f"Usage: {sys.argv[0]} [compile | test] [modules to test, concatenated with `,`]") + +MODE = sys.argv[1] +INPUT_MODULES = sys.argv[2] + +MODULES_CORE = [ + "flink-cdc-cli", + "flink-cdc-common", + "flink-cdc-composer", + "flink-cdc-runtime", + "flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base", + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values" +] + +MODULES_PIPELINE_CONNECTORS = [ + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris", + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch", + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg", + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka", + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute", + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql", + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase", + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute", + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-postgres", + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon", + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks", + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-fluss", + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-hudi", + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values" +] + +MODULES_MYSQL_SOURCE = [ + "flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc", + "flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc" +] + +MODULES_MYSQL_PIPELINE = [ + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql" +] + +MODULES_POSTGRES_SOURCE = [ + "flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc", + "flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-postgres-cdc" +] + +MODULES_POSTGRES_PIPELINE = [ + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-postgres" +] + +MODULES_ORACLE = [ + "flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc", + "flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oracle-cdc" +] + +MODULES_MONGODB = [ + "flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc", + "flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mongodb-cdc" +] + +MODULES_SQLSERVER = [ + "flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc", + "flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-sqlserver-cdc" +] + +MODULES_TIDB = [ + "flink-cdc-connect/flink-cdc-source-connectors/flink-connector-tidb-cdc", + "flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-tidb-cdc" +] + +MODULES_OCEANBASE_SOURCE = [ + "flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc", + "flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc" +] + +MODULES_OCEANBASE_PIPELINE = [ + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase" +] + +MODULES_DB2 = [ + "flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc", + "flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-db2-cdc" +] + +MODULES_VITESS = [ + "flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc", + "flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-vitess-cdc" +] + +MODULES_DORIS = [ + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris" +] + +MODULES_STARROCKS = [ + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks" +] + +MODULES_ICEBERG = [ + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg" +] + +MODULES_KAFKA = [ + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka" +] + +MODULES_PAIMON = [ + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon" +] + +MODULES_ELASTICSEARCH = [ + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch" +] + +MODULES_MAXCOMPUTE = [ + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute" +] + +MODULES_FLUSS = [ + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-fluss" +] + +MODULES_HUDI = [ + "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-hudi" +] + +MODULES_PIPELINE_E2E = [ + "flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests" +] + +MODULES_SOURCE_E2E = [ + "flink-cdc-e2e-tests/flink-cdc-source-e2e-tests" +] + +ALL_MODULES = set( + MODULES_CORE + + MODULES_PIPELINE_CONNECTORS + + MODULES_MYSQL_SOURCE + + MODULES_MYSQL_PIPELINE + + MODULES_POSTGRES_SOURCE + + MODULES_POSTGRES_PIPELINE + + MODULES_ORACLE + + MODULES_MONGODB + + MODULES_SQLSERVER + + MODULES_TIDB + + MODULES_OCEANBASE_SOURCE + + MODULES_OCEANBASE_PIPELINE + + MODULES_DB2 + + MODULES_VITESS + + MODULES_DORIS + + MODULES_STARROCKS + + MODULES_ICEBERG + + MODULES_KAFKA + + MODULES_PAIMON + + MODULES_ELASTICSEARCH + + MODULES_MAXCOMPUTE + + MODULES_FLUSS + + MODULES_HUDI + + MODULES_PIPELINE_E2E + + MODULES_SOURCE_E2E +) + +test_modules = set() +compile_modules = set() + +for module in INPUT_MODULES.split(', '): + module_list = set(globals()['MODULES_' + module.upper().replace('-', '_')]) + test_modules |= module_list + if module == 'source_e2e' or module == 'pipeline_e2e': + compile_modules |= ALL_MODULES + else: + compile_modules |= module_list + +if MODE == 'compile': + print(','.join(compile_modules)) +elif MODE == 'test': + print(','.join(test_modules)) +else: + raise RuntimeError(f"Unexpected mode: {MODE}")
