This is an automated email from the ASF dual-hosted git repository.
corgy pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 6d9e0a1755 Revert "[Improve][CI] Optimize CI" (#9971)
6d9e0a1755 is described below
commit 6d9e0a17556f780fe520e2015f767c1921be4f16
Author: Jast <[email protected]>
AuthorDate: Tue Oct 21 21:03:30 2025 +0800
Revert "[Improve][CI] Optimize CI" (#9971)
---
.github/workflows/backend.yml | 209 ++++++++++++++++-------------------
tools/github/ci_duration_analysis.py | 160 ---------------------------
tools/github/free_disk_space.sh | 124 ++++-----------------
3 files changed, 121 insertions(+), 372 deletions(-)
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 5da6224955..7121917cfa 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -61,7 +61,7 @@ jobs:
helm-chart-check:
name: Check Helm Chart Syntax
- needs: [ license-header, code-style ]
+ needs: [ license-header, code-style]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
@@ -275,18 +275,10 @@ jobs:
echo $build_modules
echo "modules=$build_modules" >> $GITHUB_OUTPUT
- delayed-scheduling:
- name: Delayed Scheduling
- needs: [ changes, sanity-check ]
- runs-on: ubuntu-latest
- steps:
- - name: Sleep for 20 seconds
- run: sleep 20s
-
dependency-license:
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
name: Dependency licenses
- needs: [ changes, sanity-check, delayed-scheduling ]
+ needs: [ changes, sanity-check ]
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
@@ -311,7 +303,7 @@ jobs:
document:
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.docs ==
'true'
- needs: [ changes, sanity-check, delayed-scheduling ]
+ needs: [ changes, sanity-check ]
name: Build website
runs-on: ubuntu-latest
timeout-minutes: 60
@@ -340,10 +332,10 @@ jobs:
seatunnel-ui:
if: needs.changes.outputs.api == 'true'
- needs: [ changes, sanity-check, delayed-scheduling ]
+ needs: [ changes, sanity-check ]
name: Build SeaTunnel UI
runs-on: ubuntu-latest
- timeout-minutes: 20
+ timeout-minutes: 60
steps:
- name: Checkout PR
uses: actions/checkout@v3
@@ -365,7 +357,7 @@ jobs:
npm run build
unit-test:
- needs: [ changes, sanity-check, delayed-scheduling ]
+ needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' || (needs.changes.outputs.api ==
'false' && needs.changes.outputs.ut-modules != '')
runs-on: ${{ matrix.os }}
strategy:
@@ -633,14 +625,14 @@ jobs:
MAVEN_OPTS: -Xmx2048m
engine-v2-it:
- needs: [ changes, sanity-check, delayed-scheduling ]
+ needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || needs.changes.outputs.engine-e2e == 'true'
runs-on: ${{ matrix.os }}
strategy:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 80
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -658,7 +650,7 @@ jobs:
MAVEN_OPTS: -Xmx4096m
engine-k8s-it:
- needs: [ changes, sanity-check, delayed-scheduling ]
+ needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' ||
contains(needs.changes.outputs.it-modules, 'seatunnel-engine-k8s-e2e')
runs-on: ${{ matrix.os }}
strategy:
@@ -702,7 +694,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 100
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -748,7 +740,7 @@ jobs:
MAVEN_OPTS: -Xmx4096m
all-connectors-it-1:
- needs: [ changes, sanity-check, delayed-scheduling ]
+ needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
env:
@@ -758,7 +750,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 60
+ timeout-minutes: 150
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -774,13 +766,12 @@ jobs:
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
run_it_modules=`python
tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules"
7 0`
- echo $run_it_modules
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$run_it_modules -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
all-connectors-it-2:
- needs: [ changes, sanity-check, delayed-scheduling]
+ needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
env:
@@ -790,7 +781,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 80
+ timeout-minutes: 150
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -806,7 +797,6 @@ jobs:
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
run_it_modules=`python
tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules"
7 1`
- echo $run_it_modules
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$run_it_modules -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -822,7 +812,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 120
+ timeout-minutes: 150
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -838,13 +828,12 @@ jobs:
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
run_it_modules=`python
tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules"
7 2`
- echo $run_it_modules
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$run_it_modules -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
all-connectors-it-4:
- needs: [ changes, sanity-check, delayed-scheduling ]
+ needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
env:
@@ -854,7 +843,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 80
+ timeout-minutes: 150
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -870,7 +859,6 @@ jobs:
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
run_it_modules=`python
tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules"
7 3`
- echo $run_it_modules
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$run_it_modules -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -886,7 +874,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 160
+ timeout-minutes: 150
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -902,7 +890,6 @@ jobs:
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
run_it_modules=`python
tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules"
7 4`
- echo $run_it_modules
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$run_it_modules -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -918,7 +905,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 140
+ timeout-minutes: 150
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -934,13 +921,12 @@ jobs:
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
run_it_modules=`python
tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules"
7 5`
- echo $run_it_modules
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$run_it_modules -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
all-connectors-it-7:
- needs: [ changes, sanity-check, delayed-scheduling ]
+ needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
env:
@@ -950,7 +936,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 100
+ timeout-minutes: 150
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -966,7 +952,6 @@ jobs:
./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
run_it_modules=`python
tools/update_modules_check/update_modules_check.py sub_it_module "$sub_modules"
7 6`
- echo $run_it_modules
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
$run_it_modules -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -982,7 +967,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 100
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -1000,7 +985,7 @@ jobs:
MAVEN_OPTS: -Xmx4096m
jdbc-connectors-it-part-2:
- needs: [ changes, sanity-check, delayed-scheduling ]
+ needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
env:
@@ -1010,7 +995,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 40
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -1028,7 +1013,7 @@ jobs:
MAVEN_OPTS: -Xmx4096m
jdbc-connectors-it-part-3:
- needs: [ changes, sanity-check, delayed-scheduling ]
+ needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
env:
@@ -1038,7 +1023,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 40
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -1056,7 +1041,7 @@ jobs:
MAVEN_OPTS: -Xmx4096m
jdbc-connectors-it-part-4:
- needs: [ changes, sanity-check, delayed-scheduling ]
+ needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
env:
@@ -1066,7 +1051,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 20
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -1084,7 +1069,7 @@ jobs:
MAVEN_OPTS: -Xmx4096m
jdbc-connectors-it-part-5:
- needs: [ changes, sanity-check, delayed-scheduling ]
+ needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
env:
@@ -1094,7 +1079,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 40
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -1112,7 +1097,7 @@ jobs:
MAVEN_OPTS: -Xmx4096m
jdbc-connectors-it-part-6:
- needs: [ changes, sanity-check, delayed-scheduling ]
+ needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
env:
@@ -1122,7 +1107,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 40
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -1168,7 +1153,7 @@ jobs:
MAVEN_OPTS: -Xmx4096m
jdbc-connectors-it-ddl:
- needs: [ changes, sanity-check, delayed-scheduling ]
+ needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
runs-on: ${{ matrix.os }}
env:
@@ -1178,7 +1163,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 40
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -1195,15 +1180,15 @@ jobs:
env:
MAVEN_OPTS: -Xmx4096m
- connector-redis-it:
+ kudu-connector-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-redis-e2e')
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-kudu-e2e')
runs-on: ${{ matrix.os }}
strategy:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 160
+ timeout-minutes: 30
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -1214,21 +1199,21 @@ jobs:
cache: 'maven'
- name: free disk space
run: tools/github/free_disk_space.sh
- - name: run redis connector integration test
+ - name: run kudu connector integration test
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-redis-e2e -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-kudu-e2e -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
- kafka-connector-it:
+ amazonSqs-connector-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-kafka-e2e')
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-amazonsqs-e2e')
runs-on: ${{ matrix.os }}
strategy:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 140
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -1239,46 +1224,21 @@ jobs:
cache: 'maven'
- name: free disk space
run: tools/github/free_disk_space.sh
- - name: run kafka connector integration test
+ - name: run amazonsqs connector integration test
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-kafka-e2e -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-amazonsqs-e2e -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
- connector-file-local-it:
+ kafka-connector-it:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-file-local-e2e')
- runs-on: ${{ matrix.os }}
- strategy:
- matrix:
- java: [ '8', '11' ]
- os: [ 'ubuntu-latest' ]
- timeout-minutes: 100
- steps:
- - uses: actions/checkout@v2
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v3
- with:
- java-version: ${{ matrix.java }}
- distribution: 'temurin'
- cache: 'maven'
- - name: free disk space
- run: tools/github/free_disk_space.sh
- - name: run file local connector integration test
- run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-file-local-e2e -am -Pci
- env:
- MAVEN_OPTS: -Xmx4096m
-
- connector-file-sftp-it:
- needs: [ changes, sanity-check, delayed-scheduling ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-file-sftp-e2e')
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-kafka-e2e')
runs-on: ${{ matrix.os }}
strategy:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 80
+ timeout-minutes: 140
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -1289,21 +1249,21 @@ jobs:
cache: 'maven'
- name: free disk space
run: tools/github/free_disk_space.sh
- - name: run file sftp connector integration test
+ - name: run kafka connector integration test
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-file-sftp-e2e -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-kafka-e2e -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
rocketmq-connector-it:
- needs: [ changes, sanity-check, delayed-scheduling ]
+ needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-rocketmq-e2e')
runs-on: ${{ matrix.os }}
strategy:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 80
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -1322,14 +1282,14 @@ jobs:
doris-connector-it:
- needs: [ changes, sanity-check, delayed-scheduling ]
+ needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-doris-e2e')
runs-on: ${{ matrix.os }}
strategy:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 60
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -1347,14 +1307,14 @@ jobs:
MAVEN_OPTS: -Xmx4096m
paimon-connector-it:
- needs: [ changes, sanity-check, delayed-scheduling ]
+ needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-paimon-e2e')
runs-on: ${{ matrix.os }}
strategy:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 60
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -1372,14 +1332,14 @@ jobs:
MAVEN_OPTS: -Xmx4096m
oracle-cdc-connector-it:
- needs: [ changes, sanity-check, delayed-scheduling ]
+ needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-cdc-oracle-e2e')
runs-on: ${{ matrix.os }}
strategy:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 40
+ timeout-minutes: 150
steps:
- name: Checkout repository
uses: actions/checkout@v2
@@ -1401,15 +1361,15 @@ jobs:
echo 'running oracle cdc connector integration test...' && \
./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-cdc-oracle-e2e -am -Pci
- kudu-connector-it:
- needs: [ changes, sanity-check, delayed-scheduling ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-kudu-e2e')
+ connector-file-local-it:
+ needs: [ changes, sanity-check ]
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-file-local-e2e')
runs-on: ${{ matrix.os }}
strategy:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 40
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -1420,21 +1380,21 @@ jobs:
cache: 'maven'
- name: free disk space
run: tools/github/free_disk_space.sh
- - name: run kudu connector integration test
+ - name: run file local connector integration test
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-kudu-e2e -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-file-local-e2e -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
- amazonSqs-connector-it:
- needs: [ changes, sanity-check, delayed-scheduling ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-amazonsqs-e2e')
+ connector-file-sftp-it:
+ needs: [ changes, sanity-check ]
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-file-sftp-e2e')
runs-on: ${{ matrix.os }}
strategy:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 30
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -1445,21 +1405,46 @@ jobs:
cache: 'maven'
- name: free disk space
run: tools/github/free_disk_space.sh
- - name: run amazonsqs connector integration test
+ - name: run file sftp connector integration test
run: |
- ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-amazonsqs-e2e -am -Pci
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-file-sftp-e2e -am -Pci
+ env:
+ MAVEN_OPTS: -Xmx4096m
+
+ connector-redis-it:
+ needs: [ changes, sanity-check ]
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules, 'connector-redis-e2e')
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ java: [ '8', '11' ]
+ os: [ 'ubuntu-latest' ]
+ timeout-minutes: 180
+ steps:
+ - uses: actions/checkout@v2
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v3
+ with:
+ java-version: ${{ matrix.java }}
+ distribution: 'temurin'
+ cache: 'maven'
+ - name: free disk space
+ run: tools/github/free_disk_space.sh
+ - name: run redis connector integration test
+ run: |
+ ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false
-D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl
:connector-redis-e2e -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
connector-sensorsdata-it:
- needs: [ changes, sanity-check, delayed-scheduling ]
- if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true' || contains(needs.changes.outputs.it-modules,
'connector-sensorsdata-e2e')
+ needs: [ changes, sanity-check ]
+ if: needs.changes.outputs.api == 'true' ||
contains(needs.changes.outputs.it-modules, 'connector-sensorsdata-e2e')
runs-on: ${{ matrix.os }}
strategy:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 30
+ timeout-minutes: 120
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
diff --git a/tools/github/ci_duration_analysis.py
b/tools/github/ci_duration_analysis.py
deleted file mode 100644
index e89eb4e341..0000000000
--- a/tools/github/ci_duration_analysis.py
+++ /dev/null
@@ -1,160 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# !/usr/bin/python
-import json
-import math
-import sys
-from datetime import datetime
-import os
-import requests
-from tabulate import tabulate
-
-
-def fetch_jobs_from_api(api_url, token=None):
- try:
- headers = {}
- if token:
- headers["Authorization"] = f"Bearer {token}"
- response = requests.get(api_url, headers=headers)
- response.raise_for_status()
- return parse_jobs_from_data(response.json())
- except requests.exceptions.RequestException as e:
- raise ConnectionError(f"API request failed: {str(e)}")
-
-
-def _parse_single_job(job):
- job_info = {
- 'job_id': job.get('id'),
- 'name': job.get('name'),
- 'status': job.get('status'),
- 'conclusion': job.get('conclusion'),
- }
-
- # Calculate the total time consumed
- try:
- start_time = datetime.fromisoformat(job['started_at'].replace('Z',
'+00:00'))
- end_time = datetime.fromisoformat(job['completed_at'].replace('Z',
'+00:00'))
- duration_seconds = (end_time - start_time).total_seconds()
- job_info['started_at'] = start_time.strftime("%Y-%m-%d %H:%M:%S")
- job_info['total_duration_seconds'] = round(duration_seconds, 2)
- job_info['total_duration_minutes'] = round(duration_seconds / 60, 2)
- except (KeyError, ValueError) as e:
- job_info['total_duration_seconds'] = None
- job_info['total_duration_minutes'] = None
- print(f"Warning: Failed to parse the time of task {job.get('id')} -
{str(e)}")
-
- # Parsing step information
- job_info['steps'] = []
- for step in job.get('steps', []):
- try:
- step_start =
datetime.fromisoformat(step['started_at'].replace('Z', '+00:00'))
- step_end =
datetime.fromisoformat(step['completed_at'].replace('Z', '+00:00'))
- step_duration = (step_end - step_start).total_seconds()
-
- job_info['steps'].append({
- 'name': step.get('name'),
- 'number': step.get('number'),
- 'status': step.get('status'),
- 'conclusion': step.get('conclusion'),
- 'duration_seconds': round(step_duration, 2)
- })
- except (KeyError, ValueError) as e:
- print(f"Warning: Failed to parse time for step {step.get('name')}
- {str(e)}")
-
- return job_info
-
-
-def _parse_jobs_batch(data):
- if 'jobs' not in data:
- raise KeyError("The data does not contain the 'jobs' field.")
- jobs = data['jobs']
- jobs.sort(
- key=lambda x: datetime.fromisoformat(x["started_at"].replace("Z",
"+00:00"))
- )
- return [_parse_single_job(job) for job in jobs]
-
-
-def parse_jobs_from_file(file_path):
- if not os.path.exists(file_path):
- raise FileNotFoundError(f"File not exist: {file_path}")
-
- with open(file_path, 'r', encoding='utf-8') as f:
- try:
- data = json.load(f)
- except json.JSONDecodeError as e:
- raise ValueError(f"JSON parsing error: {str(e)}")
-
- return _parse_jobs_batch(data)
-
-
-def parse_jobs_from_data(data):
- return _parse_jobs_batch(data)
-
-
-def print_job_summary(parsed_jobs):
- success_jobs = [job for job in parsed_jobs if job['conclusion'] ==
'success']
- print(f"A total of {len(parsed_jobs)} tasks were parsed, among which
{len(success_jobs)} tasks were successful (success)\n")
-
- job_table_data = []
- headers = ["Sequence Number", "Job Name", "Job ID", "CONCLUSION",
"STARTED_AT",
- "Total Duration (Seconds)", "Total Duration (Minutes)",
"Expected"]
-
- for i, job in enumerate(success_jobs, 1):
- remainder = job['total_duration_minutes'] % 20
- cost = (math.ceil(job['total_duration_minutes'] / 20) + 1) * 20 if
remainder >= 10 else \
- (math.ceil(job['total_duration_minutes'] / 20)) * 20
- job_table_data.append([
- i, job['name'], job['job_id'], job['conclusion'],job['started_at'],
- job['total_duration_seconds'], job['total_duration_minutes'], cost
- ])
-
- print("Overview of the Task:")
- print(tabulate(job_table_data, headers=headers, tablefmt="pipe"))
- print()
-
- for i, job in enumerate(success_jobs, 1):
- print(f"\nTask {i}: The 3 steps with the longest duration for
{job['name']}:")
- if job['steps']:
- sorted_steps = sorted(job['steps'], key=lambda x:
x['duration_seconds'], reverse=True)
- top_steps = sorted_steps[:3]
- step_table_data = [
- [s['name'], s['number'], s['status'], s['conclusion'],
s['duration_seconds']]
- for s in top_steps
- ]
- print(tabulate(step_table_data, headers=["Step name", "No",
"Status", "Conclusion", "Total Duration (seconds)"], tablefmt="pipe"))
- else:
- print("There is no step information for this task.")
- print("\n" + "-" * 80)
-
-
-if __name__ == "__main__":
- if len(sys.argv) not in (3, 4):
- print("Usage: python ci_duration_analysis.py <owner> <run_id> [token]")
- print("Usage: <owner> is a required parameter, example: hawk9821")
- print("Usage: <run_id> is a required parameter, example: 18013073919")
- print("Usage: [token] is a non-mandatory parameter. Token acquisition
method: GitHub -> Setting -> Developer Settings -> Personal access tokens ->
Tokens (classic)")
- sys.exit(1)
- owner = sys.argv[1]
- run_id = sys.argv[2]
- api_url =
f'https://api.github.com/repos/{owner}/seatunnel/actions/runs/{run_id}/jobs?page=1&per_page=100'
- token = sys.argv[3] if len(sys.argv) == 4 else None
-
- try:
- jobs_data = fetch_jobs_from_api(api_url, token)
- print_job_summary(jobs_data)
-
- except Exception as e:
- print(f"Processing failed: {str(e)}")
\ No newline at end of file
diff --git a/tools/github/free_disk_space.sh b/tools/github/free_disk_space.sh
index 18d6f168da..f9eee621f0 100755
--- a/tools/github/free_disk_space.sh
+++ b/tools/github/free_disk_space.sh
@@ -16,109 +16,33 @@
# limitations under the License.
#
-
-log_time_and_space() {
- local operation=$1
- local start_time=$2
- local start_space=$3
- # shellcheck disable=SC2155
- local end_time=$(date +%s)
- local duration=$((end_time - start_time))
- # shellcheck disable=SC2155
- local end_space=$(df -P / | tail -n 1 | awk '{print $4}')
- local freed_space=$((end_space - start_space))
- # shellcheck disable=SC2155
- local freed_gb=$(echo "scale=2; $freed_space / 1024 / 1024" | bc)
- echo
"------------------------------------------------------------------------------"
- echo "Operation: $operation"
- echo "Time taken: $duration seconds"
- echo "Freed disk space: $freed_gb GB"
- echo
"------------------------------------------------------------------------------"
- echo
-}
-
-get_available_space() {
- df -P / | tail -n 1 | awk '{print $4}'
-}
-
echo
"=============================================================================="
echo "Freeing up disk space on CI system"
echo
"=============================================================================="
-df -h
-# List 100 largest packages
-start_time=$(date +%s)
-start_space=$(get_available_space)
echo "Listing 100 largest packages"
-dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -nr | head -n 100
-
-# Uninstall the faster ones first to preload and improve overall unloading
efficiency.
-# Clean up MongoDB
-start_time=$(date +%s)
-start_space=$(get_available_space)
-sudo apt-get remove -y --purge '^mongodb-.*' > /dev/null 2>&1
-log_time_and_space "Remove MongoDB related packages" $start_time $start_space
-
-# Clean up .NET
-start_time=$(date +%s)
-start_space=$(get_available_space)
-sudo apt-get remove -y --purge '^dotnet-.*' > /dev/null 2>&1
-log_time_and_space "Remove .NET related packages" $start_time $start_space
-
-# Clean up LLVM
-start_time=$(date +%s)
-start_space=$(get_available_space)
-sudo apt-get remove -y --purge '^llvm-.*' > /dev/null 2>&1
-log_time_and_space "Remove LLVM related packages" $start_time $start_space
-
-# Clean up MySQL
-start_time=$(date +%s)
-start_space=$(get_available_space)
-sudo apt-get remove -y --purge '^mysql-.*' > /dev/null 2>&1
-log_time_and_space "Remove MySQL related packages" $start_time $start_space
-
-# Clean up large packages
-packages_to_check="ruby3.2-doc powershell azure-cli google-cloud-sdk hhvm
google-chrome-stable firefox mono-devel libgl1-mesa-dri"
-for package in $packages_to_check; do
- start_time=$(date +%s)
- start_space=$(get_available_space)
- if dpkg -l | grep -q "$package"; then
- sudo apt-get -o APT::Install-Suggests="false" remove -y --purge
"$package" > /dev/null 2>&1
- log_time_and_space "Remove $package packages" $start_time $start_space
- fi
-done
-
-# Clean up apt cache
-start_time=$(date +%s)
-start_space=$(get_available_space)
-sudo apt-get autoremove -y > /dev/null 2>&1
-sudo apt-get clean > /dev/null 2>&1
-log_time_and_space "Remove apt cache" $start_time $start_space
-
-# Clean up Android directories
-#start_time=$(date +%s)
-#start_space=$(get_available_space)
-#sudo nohup rm -rf /usr/local/lib/android > /dev/null 2>&1 &
-#log_time_and_space "Remove android directories" $start_time $start_space
-
-# Clean up large directories
-directories=(
- "/usr/local/.ghcup/"
- "/usr/share/dotnet/"
- "/usr/local/graalvm/"
- "/usr/local/share/powershell"
- "/usr/local/share/chromium"
- "/usr/local/share/boost"
- "/usr/local/lib/node_modules"
- "/opt/hostedtoolcache/CodeQL"
- "/opt/ghc"
-)
-start_time=$(date +%s)
-start_space=$(get_available_space)
-sudo bash -c 'for dir in "${@}"; do [ -d "$dir" ] && rm -rf "$dir" & done;
wait' _ "${directories[@]}"
-log_time_and_space "Remove other large directories" $start_time $start_space
-
-echo
"=============================================================================="
-echo "Disk cleanup completed"
-echo
"=============================================================================="
+dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100
+df -h
+echo "Removing large packages"
+sudo apt-get remove -y '^dotnet-.*'
+sudo apt-get remove -y '^llvm-.*'
+sudo apt-get remove -y 'php.*'
+sudo apt-get remove -y '^mongodb-.*'
+sudo apt-get remove -y '^mysql-.*'
+sudo apt-get remove -y azure-cli google-cloud-sdk hhvm google-chrome-stable
firefox powershell mono-devel libgl1-mesa-dri
+sudo apt-get autoremove -y
+sudo apt-get clean
+echo "Disk space before cleanup:"
+df -h
+echo "Removing large directories"
+sudo rm -rf /usr/share/dotnet/
+sudo rm -rf /usr/local/graalvm/
+sudo rm -rf /usr/local/.ghcup/
+sudo rm -rf /usr/local/share/powershell
+sudo rm -rf /usr/local/share/chromium
+sudo rm -rf /usr/local/share/boost
+sudo rm -rf /usr/local/lib/android
+sudo rm -rf /usr/local/lib/node_modules
+sudo rm -rf /opt/hostedtoolcache/CodeQL
+sudo rm -rf /opt/ghc
df -h
\ No newline at end of file
