This is an automated email from the ASF dual-hosted git repository.
ic4y pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new b19cebb5c [Improve][CI] split it to 4 github tasks (#3832)
b19cebb5c is described below
commit b19cebb5c82295387587c163ac0faafa0528c8b3
Author: Eric <[email protected]>
AuthorDate: Fri Dec 30 12:02:53 2022 +0800
[Improve][CI] split it to 4 github tasks (#3832)
* split it to 4 github tasks
---
.github/workflows/backend.yml | 117 +++++++++++++++++++--
seatunnel-e2e/seatunnel-e2e-common/pom.xml | 6 ++
seatunnel-e2e/seatunnel-transforms-v2-e2e/pom.xml | 1 -
tools/update_modules_check/update_modules_check.py | 11 ++
4 files changed, 127 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 2e100beca..b925f21e3 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -315,9 +315,9 @@ jobs:
env:
MAVEN_OPTS: -Xmx2048m
- integration-test:
+ updated-modules-integration-test:
needs: [ changes, sanity-check ]
- if: needs.changes.outputs.api == 'true' || (needs.changes.outputs.api ==
'false' && needs.changes.outputs.it-modules != '')
+ if: needs.changes.outputs.api == 'false' &&
needs.changes.outputs.it-modules != ''
runs-on: ${{ matrix.os }}
strategy:
matrix:
@@ -332,17 +332,120 @@ jobs:
java-version: ${{ matrix.java }}
distribution: 'temurin'
cache: 'maven'
- - name: run all modules integration test
+ - name: run updated modules integration test
+ if: needs.changes.outputs.api == 'false' &&
needs.changes.outputs.it-modules != ''
+ run: |
+ ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false
-D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true
--no-snapshot-updates -pl ${{needs.changes.outputs.it-modules}} -am -Pci
+ env:
+ MAVEN_OPTS: -Xmx2048m
+
+ engine-and-transform-v2-it:
+ needs: [ changes, sanity-check ]
+ if: needs.changes.outputs.api == 'true'
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ java: [ '8', '11' ]
+ os: [ 'ubuntu-latest' ]
+ timeout-minutes: 90
+ steps:
+ - uses: actions/checkout@v2
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v3
+ with:
+ java-version: ${{ matrix.java }}
+ distribution: 'temurin'
+ cache: 'maven'
+ - name: run some modules integration test
if: needs.changes.outputs.api == 'true'
run: |
- ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false
-D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true
--no-snapshot-updates
+ ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false
-D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true
--no-snapshot-updates -pl
:seatunnel-transforms-v2-e2e,:connector-seatunnel-e2e-base -am -Pci
env:
MAVEN_OPTS: -Xmx2048m
- - name: run updated modules integration test
- if: needs.changes.outputs.api == 'false' &&
needs.changes.outputs.it-modules != ''
+ all-connectors-it-1:
+ needs: [ changes, sanity-check ]
+ if: needs.changes.outputs.api == 'true'
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ java: [ '8', '11' ]
+ os: [ 'ubuntu-latest' ]
+ timeout-minutes: 90
+ steps:
+ - uses: actions/checkout@v2
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v3
+ with:
+ java-version: ${{ matrix.java }}
+ distribution: 'temurin'
+ cache: 'maven'
+ - name: run some modules integration test
+ if: needs.changes.outputs.api == 'true'
run: |
- ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false
-D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true
--no-snapshot-updates -pl ${{needs.changes.outputs.it-modules}} -am -Pci
+ ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-flink-connector-v2-e2e > /tmp/sub_module.txt
+ ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-spark-connector-v2-e2e >> /tmp/sub_module.txt
+ ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
+ sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
+ run_it_modules=`python
tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 3
0`
+ ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false
-D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true
--no-snapshot-updates -pl $run_it_modules -am -Pci
env:
MAVEN_OPTS: -Xmx2048m
+ all-connectors-it-2:
+ needs: [ changes, sanity-check ]
+ if: needs.changes.outputs.api == 'true'
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ java: [ '8', '11' ]
+ os: [ 'ubuntu-latest' ]
+ timeout-minutes: 90
+ steps:
+ - uses: actions/checkout@v2
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v3
+ with:
+ java-version: ${{ matrix.java }}
+ distribution: 'temurin'
+ cache: 'maven'
+ - name: run some modules integration test
+ if: needs.changes.outputs.api == 'true'
+ run: |
+ ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-flink-connector-v2-e2e > /tmp/sub_module.txt
+ ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-spark-connector-v2-e2e >> /tmp/sub_module.txt
+ ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
+ sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
+ run_it_modules=`python
tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 3
1`
+ ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false
-D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true
--no-snapshot-updates -pl $run_it_modules -am -Pci
+ env:
+ MAVEN_OPTS: -Xmx2048m
+
+ all-connectors-it-3:
+ needs: [ changes, sanity-check ]
+ if: needs.changes.outputs.api == 'true'
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ java: [ '8', '11' ]
+ os: [ 'ubuntu-latest' ]
+ timeout-minutes: 90
+ steps:
+ - uses: actions/checkout@v2
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v3
+ with:
+ java-version: ${{ matrix.java }}
+ distribution: 'temurin'
+ cache: 'maven'
+ - name: run some modules integration test
+ if: needs.changes.outputs.api == 'true'
+ run: |
+ ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-flink-connector-v2-e2e > /tmp/sub_module.txt
+ ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-spark-connector-v2-e2e >> /tmp/sub_module.txt
+ ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl :seatunnel-connector-v2-e2e >> /tmp/sub_module.txt
+ sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
+ run_it_modules=`python
tools/update_modules_check/update_modules_check.py sub_it_module $sub_modules 3
2`
+ ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false
-D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true
--no-snapshot-updates -pl $run_it_modules -am -Pci
+ env:
+ MAVEN_OPTS: -Xmx2048m
\ No newline at end of file
diff --git a/seatunnel-e2e/seatunnel-e2e-common/pom.xml
b/seatunnel-e2e/seatunnel-e2e-common/pom.xml
index 3ddc46cb5..51073e94a 100644
--- a/seatunnel-e2e/seatunnel-e2e-common/pom.xml
+++ b/seatunnel-e2e/seatunnel-e2e-common/pom.xml
@@ -31,6 +31,12 @@
<artifactId>seatunnel-api</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>seatunnel-transforms-v2</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/pom.xml
b/seatunnel-e2e/seatunnel-transforms-v2-e2e/pom.xml
index 28b459706..b86a80fbe 100644
--- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/pom.xml
+++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/pom.xml
@@ -51,7 +51,6 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
-
<dependency>
<groupId>org.apache.seatunnel</groupId>
<artifactId>connector-fake</artifactId>
diff --git a/tools/update_modules_check/update_modules_check.py
b/tools/update_modules_check/update_modules_check.py
index c573f5d9b..8a8a990a1 100644
--- a/tools/update_modules_check/update_modules_check.py
+++ b/tools/update_modules_check/update_modules_check.py
@@ -148,6 +148,15 @@ def get_deleted_modules(files):
output_module = output_module[1:len(output_module)]
print(output_module)
+def get_sub_it_modules(modules, total_num, current_num):
+ modules_arr = modules.split(",")
+ output = ""
+ for i,module in enumerate(modules_arr):
+ if len(module) > 0 and i % int(total_num) == int(current_num):
+ output = output + ",:" + module
+
+ output = output[1:len(output)]
+ print(output)
def main(argv):
if argv[1] == "cv2":
@@ -176,6 +185,8 @@ def main(argv):
get_deleted_modules(argv[2])
elif argv[1] == "rm":
remove_deleted_modules(argv[2], argv[3])
+ elif argv[1] == "sub_it_module":
+ get_sub_it_modules(argv[2], argv[3], argv[4])
if __name__ == "__main__":
