This is an automated email from the ASF dual-hosted git repository.
gaojun2048 pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 97c9fabec [Hotfix]Fix ci sub module error when only update parent
module pom.xml file (#3288)
97c9fabec is described below
commit 97c9fabecefc51258265289ee0cbba570c079a59
Author: Eric <[email protected]>
AuthorDate: Thu Nov 3 16:15:39 2022 +0800
[Hotfix]Fix ci sub module error when only update parent module pom.xml file
(#3288)
---
.github/workflows/backend.yml | 31 +++++++++-----
.github/workflows/schedule_backend.yml | 7 +---
tools/update_modules_check/update_modules_check.py | 48 ++++++++++++++++++++--
3 files changed, 66 insertions(+), 20 deletions(-)
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index b1dd0dc94..2c898080d 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -200,11 +200,16 @@ jobs:
run: |
modules='${{ steps.engine-modules.outputs.modules }}${{
steps.cv2-modules.outputs.modules }}'
modules=${modules: 1}
- includes=`python tools/update_modules_check/update_modules_check.py
tree $modules`
+ pl_modules=`python
tools/update_modules_check/update_modules_check.py replace $modules`
+ ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl $pl_modules > /tmp/sub_module.txt
+ sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
+ tree_modules="$modules$sub_modules"
+ includes=`python tools/update_modules_check/update_modules_check.py
tree $tree_modules`
./mvnw -Pno_dist -D"e2e.dependency.skip"=false dependency:tree
$includes -DoutputType=text -DoutputFile=/tmp/tree_out.txt
- modules=`python tools/update_modules_check/update_modules_check.py
final /tmp/tree_out.txt`
- echo $modules
- echo "modules=$modules" >> $GITHUB_OUTPUT
+ build_modules=`python
tools/update_modules_check/update_modules_check.py final_ut /tmp/tree_out.txt`
+ build_modules=${build_modules: 1}
+ echo $build_modules
+ echo "modules=$build_modules" >> $GITHUB_OUTPUT
- name: Make integration test modules
id: it-modules
@@ -212,11 +217,16 @@ jobs:
run: |
modules='${{ steps.cv2-e2e-modules.outputs.modules }}${{
steps.cv2-flink-e2e-modules.outputs.modules }}${{
steps.cv2-spark-e2e-modules.outputs.modules }}${{
steps.engine-e2e-modules.outputs.modules }}${{
steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'
modules=${modules: 1}
- includes=`python tools/update_modules_check/update_modules_check.py
tree $modules`
+ pl_modules=`python
tools/update_modules_check/update_modules_check.py replace $modules`
+ ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout
-pl $pl_modules > /tmp/sub_module.txt
+ sub_modules=`python
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
+ tree_modules="$modules$sub_modules"
+ includes=`python tools/update_modules_check/update_modules_check.py
tree $tree_modules`
./mvnw -Pno_dist -D"e2e.dependency.skip"=false dependency:tree
$includes -DoutputType=text -DoutputFile=/tmp/tree_out.txt
- modules=`python tools/update_modules_check/update_modules_check.py
final /tmp/tree_out.txt`
- echo $modules
- echo "modules=$modules" >> $GITHUB_OUTPUT
+ build_modules=`python
tools/update_modules_check/update_modules_check.py final_it /tmp/tree_out.txt`
+ build_modules=${build_modules: 1}
+ echo $build_modules
+ echo "modules=$build_modules" >> $GITHUB_OUTPUT
dependency-license:
if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine ==
'true'
@@ -270,12 +280,13 @@ jobs:
- name: run updated modules unit test
if: needs.changes.outputs.api == 'false' &&
needs.changes.outputs.ut-modules != ''
run: |
- ./mvnw -B -T 1C clean verify -D"maven.test.skip"=false
-D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true
--no-snapshot-updates -pl ${{needs.changes.outputs.ut-modules}} -am -amd
-Pno_dist
+ ./mvnw -B -T 1C clean verify -D"maven.test.skip"=false
-D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true
--no-snapshot-updates -pl ${{needs.changes.outputs.ut-modules}} -am -Pno_dist
env:
MAVEN_OPTS: -Xmx2048m
integration-test:
needs: [ changes, sanity-check ]
+ if: needs.changes.outputs.api == 'true' || (needs.changes.outputs.api ==
'false' && needs.changes.outputs.it-modules != '')
runs-on: ${{ matrix.os }}
strategy:
matrix:
@@ -300,7 +311,7 @@ jobs:
- name: run updated modules integration test
if: needs.changes.outputs.api == 'false' &&
needs.changes.outputs.it-modules != ''
run: |
- ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false
-D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true
--no-snapshot-updates -pl ${{needs.changes.outputs.it-modules}} -am -amd
-Pno_dist
+ ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false
-D"checkstyle.skip"=true -D"license.skipAddThirdParty"=true
--no-snapshot-updates -pl ${{needs.changes.outputs.it-modules}} -am -Pno_dist
env:
MAVEN_OPTS: -Xmx2048m
diff --git a/.github/workflows/schedule_backend.yml
b/.github/workflows/schedule_backend.yml
index 518904195..8b33e76c3 100644
--- a/.github/workflows/schedule_backend.yml
+++ b/.github/workflows/schedule_backend.yml
@@ -143,9 +143,4 @@ jobs:
./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false
-D"checkstyle.skip"=true -D"scalastyle.skip"=true
-D"license.skipAddThirdParty"=true --no-snapshot-updates
env:
MAVEN_OPTS: -Xmx2048m
- - name: run updated modules integration test
- if: needs.changes.outputs.api == 'false' &&
needs.changes.outputs.it-modules != ''
- run: |
- ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false
-D"checkstyle.skip"=true -D"scalastyle.skip"=true
-D"license.skipAddThirdParty"=true --no-snapshot-updates
- env:
- MAVEN_OPTS: -Xmx2048m
+
diff --git a/tools/update_modules_check/update_modules_check.py
b/tools/update_modules_check/update_modules_check.py
index 0e8b4f24d..c135d7c99 100644
--- a/tools/update_modules_check/update_modules_check.py
+++ b/tools/update_modules_check/update_modules_check.py
@@ -44,6 +44,10 @@ def get_modules(files, index, start_pre, root_module):
if module_name.startswith(start_pre):
modules_name_set.add(module_name)
+ sub_module_name = file.split('/')[index + 1]
+ if sub_module_name.startswith(start_pre):
+ modules_name_set.add(sub_module_name)
+
output_module = ""
if len(modules_name_set) > 0:
for module in modules_name_set:
@@ -54,6 +58,22 @@ def get_modules(files, index, start_pre, root_module):
print(output_module)
+def replace_comma_to_commacolon(modules_str):
+ modules_str = modules_str.replace(",", ",:")
+ modules_str = ":" + modules_str
+ print(modules_str)
+
+def get_sub_modules(file):
+ f = open(file, 'rb')
+ output = ""
+ for line in f.readlines():
+ line = line.replace(" ","")
+ if line.startswith("<string>"):
+ line = line.replace(" ","").replace("<string>",
"").replace("</string>", "").replace("\n", "")
+ output = output + "," + line
+
+ print(output)
+
def get_dependency_tree_includes(modules_str):
modules = modules_str.split(',')
output = ""
@@ -64,17 +84,31 @@ def get_dependency_tree_includes(modules_str):
output = "-Dincludes=" + output
print(output)
-def get_final_modules(file):
+def get_final_it_modules(file):
f = open(file, 'rb')
output = ""
for line in f.readlines():
if line.startswith("org.apache.seatunnel"):
con = line.split(":")
- if con[2] == "jar":
+ # find all e2e modules
+ if con[2] == "jar" and "-e2e" in con[1] and "transform" not in
con[1]:
output = output + "," + ":" + con[1]
+
output = output[1:len(output)]
print(output)
+def get_final_ut_modules(file):
+ f = open(file, 'rb')
+ output = ""
+ for line in f.readlines():
+ if line.startswith("org.apache.seatunnel"):
+ con = line.split(":")
+ # find all e2e modules
+ if con[2] == "jar":
+ output = output + "," + ":" + con[1]
+
+ print(output)
+
def main(argv):
if argv[1] == "cv2":
get_cv2_modules(argv[2])
@@ -90,8 +124,14 @@ def main(argv):
get_engine_e2e_modules(argv[2])
elif argv[1] == "tree":
get_dependency_tree_includes(argv[2])
- elif argv[1] == "final":
- get_final_modules(argv[2])
+ elif argv[1] == "final_it":
+ get_final_it_modules(argv[2])
+ elif argv[1] == "final_ut":
+ get_final_ut_modules(argv[2])
+ elif argv[1] == "replace":
+ replace_comma_to_commacolon(argv[2])
+ elif argv[1] == "sub":
+ get_sub_modules(argv[2])
if __name__ == "__main__":
main(sys.argv)
\ No newline at end of file
