This is an automated email from the ASF dual-hosted git repository.
dybyte pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new a80dd71582 [Improve][E2E] Rebalance connector integration test shards
(#11791)
a80dd71582 is described below
commit a80dd7158247187b9fcf504b279e39a930ab77c9
Author: zhiwei.niu <[email protected]>
AuthorDate: Wed Aug 26 02:10:00 2026 +0800
[Improve][E2E] Rebalance connector integration test shards (#11791)
---
.github/workflows/backend.yml | 150 ++++++++++------
.../test_update_modules_check.py | 198 ++++++++++++++++++---
tools/update_modules_check/update_modules_check.py | 147 ++++++++-------
3 files changed, 349 insertions(+), 146 deletions(-)
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 6f6cca79b9..c641d9d3dc 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -166,8 +166,8 @@ jobs:
benchmarks: ${{ steps.filter.outputs.benchmarks }}
docs: ${{ steps.filter.outputs.docs }}
helm: ${{ steps.filter.outputs.helm }}
- ut-modules: ${{ steps.ut-modules.outputs.modules }}
- it-modules: ${{ steps.it-modules.outputs.modules }}
+ ut-modules: ${{ steps.ut-modules.outputs.modules || '[]' }}
+ it-modules: ${{ steps.it-modules.outputs.modules || '[]' }}
steps:
- uses: actions/checkout@v4
with:
@@ -344,14 +344,16 @@ jobs:
connector_modules='${{ steps.cv2-modules.outputs.modules }}'
if [[ "zz${engine_modules}${connector_modules}" == "zz" &&
"zz"$dist_modules != "zz" ]];then
echo $pl_modules
- echo "modules=$pl_modules" >> $GITHUB_OUTPUT
+ modules_json=$(python
tools/update_modules_check/update_modules_check.py json "$pl_modules")
+ echo "modules=$modules_json" >> "$GITHUB_OUTPUT"
exit 0
fi
if [[ "zz"$connector_modules == "zz" && "zz"$engine_modules != "zz"
]];then
# Engine changes already route the full downstream engine test set
through engine=true.
# Keep the fixed smoke modules as the output without resolving the
Maven dependency tree.
echo $pl_modules
- echo "modules=$pl_modules" >> $GITHUB_OUTPUT
+ modules_json=$(python
tools/update_modules_check/update_modules_check.py json "$pl_modules")
+ echo "modules=$modules_json" >> "$GITHUB_OUTPUT"
exit 0
fi
@@ -365,7 +367,8 @@ jobs:
build_modules=$pl_modules
fi
echo $build_modules
- echo "modules=$build_modules" >> $GITHUB_OUTPUT
+ modules_json=$(python
tools/update_modules_check/update_modules_check.py json "$build_modules")
+ echo "modules=$modules_json" >> "$GITHUB_OUTPUT"
- name: Make integration test modules
id: it-modules
@@ -393,7 +396,8 @@ jobs:
# Engine changes already trigger the broad downstream integration
jobs through engine=true.
# Avoid a second Maven dependency tree pass when no connector or
engine-e2e module changed.
echo $pl_modules
- echo "modules=$pl_modules" >> $GITHUB_OUTPUT
+ modules_json=$(python
tools/update_modules_check/update_modules_check.py json "$pl_modules")
+ echo "modules=$modules_json" >> "$GITHUB_OUTPUT"
exit 0
fi
@@ -404,7 +408,8 @@ jobs:
./mvnw -Pci -D"e2e.dependency.skip"=false dependency:tree $includes
-DoutputType=text -DoutputFile=/tmp/tree_out.txt
build_modules=`python
tools/update_modules_check/update_modules_check.py final_it /tmp/tree_out.txt`
echo $build_modules
- echo "modules=$build_modules" >> $GITHUB_OUTPUT
+ modules_json=$(python
tools/update_modules_check/update_modules_check.py json "$build_modules")
+ echo "modules=$modules_json" >> "$GITHUB_OUTPUT"
dependency-license:
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
@@ -492,7 +497,7 @@ jobs:
unit-test:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' || (needs.changes.outputs.api ==
'false' && needs.changes.outputs.ut-modules != '')
+ if: needs.changes.outputs.api == 'true' || (needs.changes.outputs.api ==
'false' && needs.changes.outputs.ut-modules != '[]')
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -559,7 +564,7 @@ jobs:
updated-modules-integration-test-part-1:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine
== 'false' && needs.changes.outputs.it-modules != ''
+ if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine
== 'false' && needs.changes.outputs.it-modules != '[]'
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -579,7 +584,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run updated modules integration test (part-1)
run: |
- sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 0`
+ sub_modules=$(python
tools/update_modules_check/update_modules_check.py sub_update_it_module
"$IT_MODULES" 8 0)
if [ ! -z $sub_modules ]; then
echo $sub_modules
./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$sub_modules -am -Pci
@@ -587,11 +592,12 @@ jobs:
echo "sub modules is empty, skipping"
fi
env:
+ IT_MODULES: ${{ needs.changes.outputs.it-modules }}
MAVEN_OPTS: -Xmx4096m
updated-modules-integration-test-part-2:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine
== 'false' && needs.changes.outputs.it-modules != ''
+ if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine
== 'false' && needs.changes.outputs.it-modules != '[]'
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -611,7 +617,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run updated modules integration test (part-2)
run: |
- sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 1`
+ sub_modules=$(python
tools/update_modules_check/update_modules_check.py sub_update_it_module
"$IT_MODULES" 8 1)
if [ ! -z $sub_modules ]; then
echo $sub_modules
./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$sub_modules -am -Pci
@@ -619,11 +625,12 @@ jobs:
echo "sub modules is empty, skipping"
fi
env:
+ IT_MODULES: ${{ needs.changes.outputs.it-modules }}
MAVEN_OPTS: -Xmx4096m
updated-modules-integration-test-part-3:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine
== 'false' && needs.changes.outputs.it-modules != ''
+ if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine
== 'false' && needs.changes.outputs.it-modules != '[]'
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -643,7 +650,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run updated modules integration test (part-3)
run: |
- sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 2`
+ sub_modules=$(python
tools/update_modules_check/update_modules_check.py sub_update_it_module
"$IT_MODULES" 8 2)
if [ ! -z $sub_modules ]; then
echo $sub_modules
./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$sub_modules -am -Pci
@@ -651,11 +658,12 @@ jobs:
echo "sub modules is empty, skipping"
fi
env:
+ IT_MODULES: ${{ needs.changes.outputs.it-modules }}
MAVEN_OPTS: -Xmx4096m
updated-modules-integration-test-part-4:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine
== 'false' && needs.changes.outputs.it-modules != ''
+ if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine
== 'false' && needs.changes.outputs.it-modules != '[]'
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -675,7 +683,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run updated modules integration test (part-4)
run: |
- sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 3`
+ sub_modules=$(python
tools/update_modules_check/update_modules_check.py sub_update_it_module
"$IT_MODULES" 8 3)
if [ ! -z $sub_modules ]; then
echo $sub_modules
./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$sub_modules -am -Pci
@@ -683,10 +691,11 @@ jobs:
echo "sub modules is empty, skipping"
fi
env:
+ IT_MODULES: ${{ needs.changes.outputs.it-modules }}
MAVEN_OPTS: -Xmx4096m
updated-modules-integration-test-part-5:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine
== 'false' && needs.changes.outputs.it-modules != ''
+ if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine
== 'false' && needs.changes.outputs.it-modules != '[]'
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -706,7 +715,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run updated modules integration test (part-5)
run: |
- sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 4`
+ sub_modules=$(python
tools/update_modules_check/update_modules_check.py sub_update_it_module
"$IT_MODULES" 8 4)
if [ ! -z $sub_modules ]; then
echo $sub_modules
./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$sub_modules -am -Pci
@@ -714,10 +723,11 @@ jobs:
echo "sub modules is empty, skipping"
fi
env:
+ IT_MODULES: ${{ needs.changes.outputs.it-modules }}
MAVEN_OPTS: -Xmx4096m
updated-modules-integration-test-part-6:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine
== 'false' && needs.changes.outputs.it-modules != ''
+ if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine
== 'false' && needs.changes.outputs.it-modules != '[]'
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -737,7 +747,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run updated modules integration test (part-6)
run: |
- sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 5`
+ sub_modules=$(python
tools/update_modules_check/update_modules_check.py sub_update_it_module
"$IT_MODULES" 8 5)
if [ ! -z $sub_modules ]; then
echo $sub_modules
./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$sub_modules -am -Pci
@@ -745,10 +755,11 @@ jobs:
echo "sub modules is empty, skipping"
fi
env:
+ IT_MODULES: ${{ needs.changes.outputs.it-modules }}
MAVEN_OPTS: -Xmx4096m
updated-modules-integration-test-part-7:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine
== 'false' && needs.changes.outputs.it-modules != ''
+ if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine
== 'false' && needs.changes.outputs.it-modules != '[]'
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -768,7 +779,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run updated modules integration test (part-7)
run: |
- sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 6`
+ sub_modules=$(python
tools/update_modules_check/update_modules_check.py sub_update_it_module
"$IT_MODULES" 8 6)
if [ ! -z $sub_modules ]; then
echo $sub_modules
./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$sub_modules -am -Pci
@@ -776,11 +787,12 @@ jobs:
echo "sub modules is empty, skipping"
fi
env:
+ IT_MODULES: ${{ needs.changes.outputs.it-modules }}
MAVEN_OPTS: -Xmx4096m
updated-modules-integration-test-part-8:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine
== 'false' && needs.changes.outputs.it-modules != ''
+ if: needs.changes.outputs.api == 'false' && needs.changes.outputs.engine
== 'false' && needs.changes.outputs.it-modules != '[]'
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -800,7 +812,7 @@ jobs:
run: tools/github/free_disk_space.sh
- name: run updated modules integration test (part-8)
run: |
- sub_modules=`python
tools/update_modules_check/update_modules_check.py sub_update_it_module
${{needs.changes.outputs.it-modules}} 8 7`
+ sub_modules=$(python
tools/update_modules_check/update_modules_check.py sub_update_it_module
"$IT_MODULES" 8 7)
if [ ! -z $sub_modules ]; then
echo $sub_modules
./mvnw -T 1 -B verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$sub_modules -am -Pci
@@ -808,6 +820,7 @@ jobs:
echo "sub modules is empty, skipping"
fi
env:
+ IT_MODULES: ${{ needs.changes.outputs.it-modules }}
MAVEN_OPTS: -Xmx4096m
engine-v2-it:
@@ -838,7 +851,7 @@ jobs:
edge-agent-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.edge-agent == 'true' ||
needs.changes.outputs.edge-agent-e2e == 'true'
+ if: needs.changes.outputs.edge-agent == 'true' ||
needs.changes.outputs.edge-agent-e2e == 'true' ||
contains(fromJSON(needs.changes.outputs.it-modules), 'seatunnel-edge-agent-e2e')
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -864,7 +877,7 @@ jobs:
engine-k8s-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' ||
contains(needs.changes.outputs.it-modules, 'seatunnel-engine-k8s-e2e')
+ if: needs.changes.outputs.api == 'true' ||
contains(fromJSON(needs.changes.outputs.it-modules), 'seatunnel-engine-k8s-e2e')
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -955,6 +968,8 @@ jobs:
env:
MAVEN_OPTS: -Xmx4096m
+ # Dedicated connector ownership and shard rebalancing guidance are
maintained in
+ # tools/update_modules_check/update_modules_check.py.
all-connectors-it-1:
needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
@@ -967,7 +982,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 210
+ timeout-minutes: 150
steps:
- uses: actions/checkout@v4
- name: Set up JDK ${{ matrix.java }}
@@ -999,7 +1014,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 180
+ timeout-minutes: 150
steps:
- uses: actions/checkout@v4
- name: Set up JDK ${{ matrix.java }}
@@ -1031,7 +1046,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 240
+ timeout-minutes: 150
steps:
- uses: actions/checkout@v4
- name: Set up JDK ${{ matrix.java }}
@@ -1063,7 +1078,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 270
+ timeout-minutes: 150
steps:
- uses: actions/checkout@v4
- name: Set up JDK ${{ matrix.java }}
@@ -1095,7 +1110,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 270
+ timeout-minutes: 150
steps:
- uses: actions/checkout@v4
- name: Set up JDK ${{ matrix.java }}
@@ -1127,7 +1142,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 210
+ timeout-minutes: 150
steps:
- uses: actions/checkout@v4
- name: Set up JDK ${{ matrix.java }}
@@ -1159,7 +1174,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 210
+ timeout-minutes: 150
steps:
- uses: actions/checkout@v4
- name: Set up JDK ${{ matrix.java }}
@@ -1179,9 +1194,9 @@ jobs:
env:
MAVEN_OPTS: -Xmx4096m
- all-connectors-it-8:
+ iceberg-connector-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(fromJSON(needs.changes.outputs.it-modules),
'connector-iceberg-e2e')
runs-on: ${{ matrix.os }}
env:
RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
@@ -1191,7 +1206,36 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 210
+ timeout-minutes: 150
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v4
+ with:
+ java-version: ${{ matrix.java }}
+ distribution: 'temurin'
+ cache: 'maven'
+ - name: free disk space
+ run: tools/github/free_disk_space.sh
+ - name: run iceberg connector integration test
+ run: |
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-iceberg-e2e -am -Pci
+ env:
+ MAVEN_OPTS: -Xmx4096m
+
+ hbase-connector-it:
+ needs: [ changes, sanity-check ]
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(fromJSON(needs.changes.outputs.it-modules),
'connector-hbase-e2e')
+ runs-on: ${{ matrix.os }}
+ env:
+ RUN_ALL_CONTAINER: ${{ needs.changes.outputs.api }}
+ RUN_ZETA_CONTAINER: ${{ needs.changes.outputs.engine }}
+ strategy:
+ fail-fast: false
+ matrix:
+ java: [ '8', '11' ]
+ os: [ 'ubuntu-latest' ]
+ timeout-minutes: 150
steps:
- uses: actions/checkout@v4
- name: Set up JDK ${{ matrix.java }}
@@ -1202,10 +1246,9 @@ jobs:
cache: 'maven'
- name: free disk space
run: tools/github/free_disk_space.sh
- - name: run connector-v2 integration test (part-8)
+ - name: run hbase connector integration test
run: |
- # These two suites dominated part-2 and triggered hosted-runner
heartbeat loss.
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-iceberg-e2e,:connector-hbase-e2e -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-hbase-e2e -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -1443,7 +1486,7 @@ jobs:
kudu-connector-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-kudu-e2e')
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(fromJSON(needs.changes.outputs.it-modules),
'connector-kudu-e2e')
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -1470,7 +1513,7 @@ jobs:
amazonSqs-connector-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-amazonsqs-e2e')
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(fromJSON(needs.changes.outputs.it-modules),
'connector-amazonsqs-e2e')
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -1496,7 +1539,7 @@ jobs:
google-pubsub-connector-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules,
'connector-google-pubsub-e2e')
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(fromJSON(needs.changes.outputs.it-modules),
'connector-google-pubsub-e2e')
runs-on: ${{ matrix.os }}
strategy:
matrix:
@@ -1521,7 +1564,7 @@ jobs:
kafka-connector-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-kafka-e2e')
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(fromJSON(needs.changes.outputs.it-modules),
'connector-kafka-e2e')
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -1547,7 +1590,7 @@ jobs:
rocketmq-connector-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-rocketmq-e2e')
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(fromJSON(needs.changes.outputs.it-modules),
'connector-rocketmq-e2e')
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -1573,7 +1616,7 @@ jobs:
elasticsearch-connector-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules,
'connector-elasticsearch-e2e')
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(fromJSON(needs.changes.outputs.it-modules),
'connector-elasticsearch-e2e')
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -1599,7 +1642,7 @@ jobs:
mysql-cdc-connector-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-cdc-mysql-e2e')
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(fromJSON(needs.changes.outputs.it-modules),
'connector-cdc-mysql-e2e')
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -1623,10 +1666,9 @@ jobs:
env:
MAVEN_OPTS: -Xmx4096m
-
doris-connector-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-doris-e2e')
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(fromJSON(needs.changes.outputs.it-modules),
'connector-doris-e2e')
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -1652,7 +1694,7 @@ jobs:
paimon-connector-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-paimon-e2e')
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(fromJSON(needs.changes.outputs.it-modules),
'connector-paimon-e2e')
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -1678,7 +1720,7 @@ jobs:
oracle-cdc-connector-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-cdc-oracle-e2e')
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(fromJSON(needs.changes.outputs.it-modules),
'connector-cdc-oracle-e2e')
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -1709,7 +1751,7 @@ jobs:
connector-file-local-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-file-local-e2e')
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(fromJSON(needs.changes.outputs.it-modules),
'connector-file-local-e2e')
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -1735,7 +1777,7 @@ jobs:
connector-file-sftp-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-file-sftp-e2e')
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(fromJSON(needs.changes.outputs.it-modules),
'connector-file-sftp-e2e')
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -1761,7 +1803,7 @@ jobs:
connector-redis-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-redis-e2e')
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(fromJSON(needs.changes.outputs.it-modules),
'connector-redis-e2e')
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -1787,7 +1829,7 @@ jobs:
connector-sensorsdata-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' ||
contains(needs.changes.outputs.it-modules, 'connector-sensorsdata-e2e')
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(fromJSON(needs.changes.outputs.it-modules),
'connector-sensorsdata-e2e')
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
diff --git a/tools/update_modules_check/test_update_modules_check.py
b/tools/update_modules_check/test_update_modules_check.py
index 59c4f2e13f..341979243d 100644
--- a/tools/update_modules_check/test_update_modules_check.py
+++ b/tools/update_modules_check/test_update_modules_check.py
@@ -1,4 +1,4 @@
-#
+#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -14,31 +14,153 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""Regression tests for connector E2E module sharding."""
+
+import io
+import re
import unittest
from collections import Counter
+from contextlib import redirect_stdout
from pathlib import Path
-import re
from update_modules_check import (
ALL_CONNECTORS_DEDICATED_SHARD_MODULES,
+ ALL_CONNECTORS_OPTIONAL_DEDICATED_SHARD_MODULES,
ALL_CONNECTORS_REQUIRED_DEDICATED_SHARD_MODULES,
build_sub_it_modules,
+ get_sub_it_modules,
+ get_sub_update_it_modules,
+ modules_to_json,
+ split_full_connector_it_modules,
)
-class UpdateModulesCheckTest(unittest.TestCase):
- """
- Guard the all-connectors shard contract used by backend CI.
- """
+class ConnectorItShardingTest(unittest.TestCase):
+ """Verify connector sharding remains stable as modules change."""
@staticmethod
def parse_modules(modules):
return [module.lstrip(":") for module in modules.split(",") if module]
- def test_regular_shards_keep_only_remaining_modules_once(self):
- """
- Regular all-connectors shards should keep each surviving module once.
- """
+ @staticmethod
+ def workflow_text():
+ return (
+ Path(__file__).resolve().parents[2] / ".github" / "workflows" /
"backend.yml"
+ ).read_text(encoding="utf-8")
+
+ def test_modules_to_json_preserves_exact_module_tokens(self) -> None:
+ self.assertEqual(
+ '["connector-kafka-e2e", "connector-iceberg-e2e"]',
+ modules_to_json(":connector-kafka-e2e,:connector-iceberg-e2e"),
+ )
+ self.assertEqual("[]", modules_to_json(""))
+
+ def test_module_outputs_use_json_empty_array(self) -> None:
+ workflow = self.workflow_text()
+ self.assertIn(
+ "ut-modules: ${{ steps.ut-modules.outputs.modules || '[]' }}",
workflow
+ )
+ self.assertIn(
+ "it-modules: ${{ steps.it-modules.outputs.modules || '[]' }}",
workflow
+ )
+ self.assertIn("needs.changes.outputs.ut-modules != '[]'", workflow)
+ self.assertIn("needs.changes.outputs.it-modules != '[]'", workflow)
+
+ def test_every_module_is_assigned_once(self) -> None:
+ modules = ["connector-a-e2e", "connector-b-e2e", "connector-c-e2e"]
+
+ shards = split_full_connector_it_modules(modules, 7)
+
+ assigned_modules = [module for shard in shards for module in shard]
+ self.assertCountEqual(modules, assigned_modules)
+ self.assertEqual(len(modules), len(assigned_modules))
+
+ def test_unknown_module_does_not_reshuffle_existing_modules(self) -> None:
+ modules = ["connector-a-e2e", "connector-b-e2e", "connector-c-e2e"]
+ original_shards = split_full_connector_it_modules(modules, 7)
+
+ shards_with_new_module = split_full_connector_it_modules(
+ modules + ["new-connector-e2e"], 7
+ )
+
+ for original_shard, new_shard in zip(original_shards,
shards_with_new_module):
+ self.assertEqual(
+ original_shard,
+ [module for module in new_shard if module !=
"new-connector-e2e"],
+ )
+
+ def test_sharding_is_independent_of_module_order(self) -> None:
+ modules = ["connector-a-e2e", "connector-b-e2e", "connector-c-e2e"]
+
+ self.assertEqual(
+ split_full_connector_it_modules(modules, 7),
+ split_full_connector_it_modules(list(reversed(modules)), 7),
+ )
+
+ def test_historical_seed_assignments_are_preserved(self) -> None:
+ modules = [
+ "connector-file-hadoop-e2e",
+ "connector-cdc-mongodb-e2e",
+ "connector-clickhouse-e2e",
+ "connector-typesense-e2e",
+ "connector-file-ftp-e2e",
+ "connector-databend-e2e",
+ "connector-http-e2e",
+ ]
+
+ self.assertEqual(
+ split_full_connector_it_modules(modules, 7),
+ [
+ ["connector-file-hadoop-e2e"],
+ ["connector-cdc-mongodb-e2e"],
+ ["connector-clickhouse-e2e"],
+ ["connector-typesense-e2e"],
+ ["connector-file-ftp-e2e"],
+ ["connector-databend-e2e"],
+ ["connector-http-e2e"],
+ ],
+ )
+
+ def test_full_and_updated_paths_apply_their_ownership_rules(self) -> None:
+ self.assertIn(
+ "connector-google-pubsub-e2e",
+ ALL_CONNECTORS_REQUIRED_DEDICATED_SHARD_MODULES,
+ )
+ connector_modules = [
+ "connector-normal-e2e",
+ *ALL_CONNECTORS_REQUIRED_DEDICATED_SHARD_MODULES,
+ *ALL_CONNECTORS_OPTIONAL_DEDICATED_SHARD_MODULES,
+ "connector-iceberg-hadoop3-e2e",
+ "connector-iceberg-s3-e2e",
+ ]
+ updated_modules = connector_modules + [
+ "seatunnel-engine-k8s-e2e",
+ ]
+
+ full_output = io.StringIO()
+ with redirect_stdout(full_output):
+ get_sub_it_modules("," + ",".join(connector_modules), 1, 0)
+ self.assertEqual(
+ full_output.getvalue(),
+ ":connector-iceberg-hadoop3-e2e,"
+ ":connector-iceberg-s3-e2e,"
+ ":connector-normal-e2e\n",
+ )
+
+ updated_output = io.StringIO()
+ with redirect_stdout(updated_output):
+ get_sub_update_it_modules(
+ modules_to_json(":" + ",:".join(updated_modules)), 1, 0
+ )
+ self.assertEqual(
+ updated_output.getvalue(),
+ ":connector-normal-e2e,"
+ ":connector-jdbc-e2e,"
+ ":connector-iceberg-hadoop3-e2e,"
+ ":connector-iceberg-s3-e2e\n",
+ )
+
+ def test_regular_shards_keep_only_remaining_modules_once(self) -> None:
expected_modules = {
"connector-assert-e2e",
"connector-cdc-sqlserver-e2e",
@@ -59,14 +181,8 @@ class UpdateModulesCheckTest(unittest.TestCase):
self.assertTrue(
set(ALL_CONNECTORS_DEDICATED_SHARD_MODULES).isdisjoint(set(combined_counter))
)
- for output_modules in shard_modules:
- self.assertNotIn("connector-iceberg-e2e", output_modules)
- self.assertNotIn("connector-hbase-e2e", output_modules)
- def test_regular_shards_fail_fast_when_dedicated_modules_disappear(self):
- """
- The all-connectors source list should fail loudly if a dedicated
module drifts.
- """
+ def test_regular_shards_fail_fast_when_dedicated_modules_disappear(self)
-> None:
modules = ",".join(
[
"",
@@ -82,10 +198,9 @@ class UpdateModulesCheckTest(unittest.TestCase):
with self.assertRaisesRegex(ValueError, "connector-elasticsearch-e2e"):
build_sub_it_modules(modules, 7, 0)
- def
test_regular_shards_allow_optional_dedicated_modules_to_be_absent(self):
- """
- Dedicated suites outside connector-v2 input should stay optional.
- """
+ def test_regular_shards_allow_optional_dedicated_modules_to_be_absent(
+ self,
+ ) -> None:
modules = ",".join(
["", "connector-assert-e2e",
*ALL_CONNECTORS_REQUIRED_DEDICATED_SHARD_MODULES]
)
@@ -99,16 +214,11 @@ class UpdateModulesCheckTest(unittest.TestCase):
self.assertEqual({"connector-assert-e2e"}, combined_modules)
- def test_workflow_keeps_dedicated_jobs_for_excluded_modules(self):
- """
- Workflow job lists should continue covering modules excluded from
regular shards.
- """
- workflow = (
- Path(__file__).resolve().parents[2] / ".github" / "workflows" /
"backend.yml"
- ).read_text(encoding="utf-8")
+ def test_workflow_keeps_dedicated_jobs_for_excluded_modules(self) -> None:
workflow_modules = set()
for modules in re.findall(
- r"-pl\s+(:[A-Za-z0-9._-]+(?:,:[A-Za-z0-9._-]+)*)", workflow
+ r"-pl\s+(:[A-Za-z0-9._-]+(?:,:[A-Za-z0-9._-]+)*)",
+ self.workflow_text(),
):
workflow_modules.update(
module.lstrip(":") for module in modules.split(",") if module
@@ -134,6 +244,36 @@ class UpdateModulesCheckTest(unittest.TestCase):
f"Missing dedicated workflow modules:
{sorted(expected_workflow_modules - workflow_modules)}",
)
+ def test_dedicated_job_conditions_match_json_module_tokens(self) -> None:
+ workflow = self.workflow_text()
+ condition_modules =
set(ALL_CONNECTORS_REQUIRED_DEDICATED_SHARD_MODULES) - {
+ "connector-jdbc-e2e"
+ }
+ condition_modules.update(
+ {"seatunnel-edge-agent-e2e", "seatunnel-engine-k8s-e2e"}
+ )
+ for module in sorted(condition_modules):
+ with self.subTest(module=module):
+ self.assertIn(
+ "contains(fromJSON(needs.changes.outputs.it-modules), "
+ f"'{module}')",
+ workflow,
+ )
+
+ def test_full_shard_rejects_non_positive_shard_count(self) -> None:
+ with self.assertRaisesRegex(
+ ValueError, "total shard count must be positive, got 0"
+ ):
+ build_sub_it_modules("connector-normal-e2e", 0, 0)
+
+ def test_full_shard_rejects_out_of_range_index(self) -> None:
+ for current_num in (-1, 7):
+ with self.subTest(current_num=current_num):
+ with self.assertRaisesRegex(
+ ValueError,
+ f"shard index {current_num} out of range \\[0, 7\\)",
+ ):
+ build_sub_it_modules("connector-normal-e2e", 7,
current_num)
if __name__ == "__main__":
unittest.main()
diff --git a/tools/update_modules_check/update_modules_check.py
b/tools/update_modules_check/update_modules_check.py
index 844bd6c48a..e9217d5bf4 100644
--- a/tools/update_modules_check/update_modules_check.py
+++ b/tools/update_modules_check/update_modules_check.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -13,27 +14,45 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# !/usr/bin/python
import json
import sys
+import zlib
+
+# Selected from the module durations in GitHub Actions run 31611542978, where
+# the resulting seven shards were estimated at 79.1-89.6 minutes. The stable
+# hash keeps existing modules in the same shard when modules are added or
removed,
+# without maintaining per-module durations or assignments.
+#
+# Re-tune only when the module set or durations change enough to unbalance the
+# shards. Export the latest per-module durations as {module: seconds}, evaluate
+# candidate seeds by assigning each module with the crc32 expression below, and
+# choose the seed that minimizes (maximum shard duration, shard-duration
spread).
+# Update test_historical_seed_assignments_are_preserved with the seed so an
+# accidental reshuffle cannot silently change the CI matrix.
+_FULL_CONNECTOR_IT_SHARD_SEED = "37709"
+
+# Connector modules handled by jobs outside the shared connector shards. Add a
+# module here only after its dedicated job handles API, engine, and direct
changes
+# to that module.
ALL_CONNECTORS_REQUIRED_DEDICATED_SHARD_MODULES = (
"connector-jdbc-e2e",
"connector-kafka-e2e",
"connector-rocketmq-e2e",
"connector-kudu-e2e",
"connector-amazonsqs-e2e",
+ "connector-google-pubsub-e2e",
"connector-doris-e2e",
"connector-paimon-e2e",
"connector-cdc-oracle-e2e",
"connector-file-local-e2e",
"connector-file-sftp-e2e",
"connector-redis-e2e",
- "connector-sensorsdata-e2e",
"connector-elasticsearch-e2e",
"connector-cdc-mysql-e2e",
"connector-iceberg-e2e",
"connector-hbase-e2e",
+ "connector-sensorsdata-e2e",
)
# These suites have dedicated jobs in backend.yml, but they are not listed by
@@ -49,6 +68,12 @@ ALL_CONNECTORS_DEDICATED_SHARD_MODULES = (
+ ALL_CONNECTORS_OPTIONAL_DEDICATED_SHARD_MODULES
)
+# The JDBC aggregate is excluded from full shards because its dedicated part
+# jobs own full runs, but direct JDBC changes still use the updated leaf
shards.
+_CONNECTOR_IT_MODULES_WITH_DEDICATED_JOB = set(
+ ALL_CONNECTORS_REQUIRED_DEDICATED_SHARD_MODULES
+) - {"connector-jdbc-e2e"}
+
def get_cv2_modules(files):
get_modules(files, 1, "connector-", "seatunnel-connectors-v2")
@@ -96,6 +121,12 @@ def replace_comma_to_commacolon(modules_str):
print(modules_str)
+def modules_to_json(modules):
+ return json.dumps(
+ [module.lstrip(":") for module in modules.split(",") if module]
+ )
+
+
def get_sub_modules(file):
output = ""
with open(file, 'r', encoding='utf-8') as f:
@@ -173,14 +204,20 @@ def get_deleted_modules(files):
print(output_module)
-def filter_dedicated_shard_modules(modules_arr, dedicated_modules,
fail_on_missing):
- """
- Remove modules that already run in dedicated workflow jobs.
+def _filter_shared_it_modules(modules, extra_exclusions=()):
+ excluded_modules = _CONNECTOR_IT_MODULES_WITH_DEDICATED_JOB | set(
+ extra_exclusions
+ )
+ return [
+ module
+ for module in dict.fromkeys(modules)
+ if module and module not in excluded_modules
+ ]
+
- The all-connectors path stays strict because a missing dedicated module
- means the exclusion list drifted away from backend.yml.
- """
- module_set = set(modules_arr)
+def filter_dedicated_shard_modules(modules, dedicated_modules,
fail_on_missing):
+ """Remove modules owned by dedicated jobs and optionally detect workflow
drift."""
+ module_set = set(modules)
if fail_on_missing:
missing_modules = [
module for module in dedicated_modules if module not in module_set
@@ -192,29 +229,41 @@ def filter_dedicated_shard_modules(modules_arr,
dedicated_modules, fail_on_missi
)
dedicated_modules_set = set(dedicated_modules)
- return [module for module in modules_arr if module not in
dedicated_modules_set]
+ return [module for module in modules if module not in
dedicated_modules_set]
-def build_sub_it_modules(modules, total_num, current_num):
- """
- Build one all-connectors shard while excluding suites with dedicated jobs.
+def split_full_connector_it_modules(modules, total_num):
+ if total_num <= 0:
+ raise ValueError(f"total shard count must be positive, got
{total_num}")
+
+ shards = [[] for _ in range(total_num)]
+ for module in sorted(set(modules)):
+ shard_key = f"{_FULL_CONNECTOR_IT_SHARD_SEED}:{module}".encode("utf-8")
+ shard = zlib.crc32(shard_key) % total_num
+ shards[shard].append(module)
+ return shards
+
- Heavy suites that already have their own workflow shard must stay out of
the
- round-robin shards, otherwise CI runs them twice and wastes runner time.
- """
- modules_arr = list(dict.fromkeys(modules.split(",")))
+def build_sub_it_modules(modules, total_num, current_num):
+ """Build one stable full connector shard after applying ownership
exclusions."""
+ total_num = int(total_num)
+ current_num = int(current_num)
+ if total_num <= 0:
+ raise ValueError(f"total shard count must be positive, got
{total_num}")
+ if not 0 <= current_num < total_num:
+ raise ValueError(
+ f"shard index {current_num} out of range [0, {total_num})"
+ )
+
+ modules_arr = [module for module in dict.fromkeys(modules.split(",")) if
module]
modules_arr = filter_dedicated_shard_modules(
modules_arr, ALL_CONNECTORS_REQUIRED_DEDICATED_SHARD_MODULES, True
)
modules_arr = filter_dedicated_shard_modules(
modules_arr, ALL_CONNECTORS_OPTIONAL_DEDICATED_SHARD_MODULES, False
)
- output = []
- for i, module in enumerate(modules_arr):
- if len(module) > 0 and i % int(total_num) == int(current_num):
- output.append(":" + module)
-
- return ",".join(output)
+ shards = split_full_connector_it_modules(modules_arr, total_num)
+ return ",".join(":" + module for module in shards[current_num])
def get_sub_it_modules(modules, total_num, current_num):
@@ -223,46 +272,16 @@ def get_sub_it_modules(modules, total_num, current_num):
def get_sub_update_it_modules(modules, total_num, current_num):
final_modules = list()
- # :connector-jdbc-e2e-common,:connector-jdbc-e2e-part-1 -->
connector-jdbc-e2e-common,:connector-jdbc-e2e-part-1
- modules = modules[1:]
- # connector-jdbc-e2e-common,:connector-jdbc-e2e-part-1 -->
[connector-jdbc-e2e-common, connector-jdbc-e2e-part-1]
- module_list = list(dict.fromkeys(modules.split(",:")))
- if "connector-kudu-e2e" in module_list:
- module_list.remove("connector-kudu-e2e")
- if "connector-amazonsqs-e2e" in module_list:
- module_list.remove("connector-amazonsqs-e2e")
- if "connector-kafka-e2e" in module_list:
- module_list.remove("connector-kafka-e2e")
- if "connector-rocketmq-e2e" in module_list:
- module_list.remove("connector-rocketmq-e2e")
- if "seatunnel-engine-k8s-e2e" in module_list:
- module_list.remove("seatunnel-engine-k8s-e2e")
- if "connector-seatunnel-e2e-base" in module_list:
- module_list.remove("connector-seatunnel-e2e-base")
- if "connector-console-seatunnel-e2e" in module_list:
- module_list.remove("connector-console-seatunnel-e2e")
- if "connector-doris-e2e" in module_list:
- module_list.remove("connector-doris-e2e")
- if "connector-paimon-e2e" in module_list:
- module_list.remove("connector-paimon-e2e")
- if "connector-cdc-oracle-e2e" in module_list:
- module_list.remove("connector-cdc-oracle-e2e")
- if "connector-file-local-e2e" in module_list:
- module_list.remove("connector-file-local-e2e")
- if "connector-file-sftp-e2e" in module_list:
- module_list.remove("connector-file-sftp-e2e")
- if "connector-redis-e2e" in module_list:
- module_list.remove("connector-redis-e2e")
- if "connector-elasticsearch-e2e" in module_list:
- module_list.remove("connector-elasticsearch-e2e")
- if "connector-cdc-mysql-e2e" in module_list:
- module_list.remove("connector-cdc-mysql-e2e")
- if "connector-seatunnel-e2e-base" in module_list:
- module_list.remove("connector-seatunnel-e2e-base")
- if "connector-console-seatunnel-e2e" in module_list:
- module_list.remove("connector-console-seatunnel-e2e")
- if "seatunnel-edge-agent-e2e" in module_list:
- module_list.remove("seatunnel-edge-agent-e2e")
+ module_names = json.loads(modules)
+ module_list = _filter_shared_it_modules(
+ module_names,
+ {
+ "connector-seatunnel-e2e-base",
+ "connector-console-seatunnel-e2e",
+ "seatunnel-engine-k8s-e2e",
+ "seatunnel-edge-agent-e2e",
+ },
+ )
for i, module in enumerate(module_list):
if len(module) > 0 and i % int(total_num) == int(current_num):
final_modules.append(":" + module)
@@ -286,6 +305,8 @@ def main(argv):
get_final_ut_modules(argv[2])
elif argv[1] == "replace":
replace_comma_to_commacolon(argv[2])
+ elif argv[1] == "json":
+ print(modules_to_json(argv[2]))
elif argv[1] == "sub":
get_sub_modules(argv[2])
elif argv[1] == "delete":
