[incubator-seatunnel] branch test_ci_1 updated: fix ci error when remove modules (#3399)

[gaojun2048]

This is an automated email from the ASF dual-hosted git repository.

gaojun2048 pushed a commit to branch test_ci_1
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git


The following commit(s) were added to refs/heads/test_ci_1 by this push:
     new 7532da75c fix ci error when remove modules (#3399)
7532da75c is described below

commit 7532da75cf2a3b3c1109818a3511d4a6697e25e6
Author: Eric <gaojun2...@gmail.com>
AuthorDate: Fri Nov 11 23:30:12 2022 +0800

    fix ci error when remove modules (#3399)
---
 .github/workflows/backend.yml                      | 21 +++++++++++++++
 tools/update_modules_check/update_modules_check.py | 31 ++++++++++++++++++++++
 2 files changed, 52 insertions(+)

diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index a7d3865e7..086bf6059 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -138,6 +138,8 @@ jobs:
               - "seatunnel-engine/**"
             engine-e2e:
               - "seatunnel-e2e/seatunnel-engine-e2e/**"
+            deleted-poms:
+              - deleted: "**/pom.xml" 
 
       - name: Check Connector V2 Update
         id: cv2-modules
@@ -193,6 +195,15 @@ jobs:
           echo $modules
           echo "modules=$modules" >> $GITHUB_OUTPUT
 
+      - name: Check Deleted Modules
+        id: deleted-modules
+        if: ${{ steps.filter.outputs.deleted-poms == 'true' }}
+        run: |
+          update_files='${{ steps.filter.outputs.deleted-poms_files }}'
+          modules=`python tools/update_modules_check/update_modules_check.py 
delete $update_files`
+          echo $modules
+          echo "modules=$modules" >> $GITHUB_OUTPUT      
+
       - name: Make unit test modules
         id: ut-modules
         if: ${{ steps.filter.outputs.api == 'false' && 
(steps.engine-modules.outputs.modules != '' || 
steps.cv2-modules.outputs.modules != '') }}
@@ -200,6 +211,11 @@ jobs:
           modules='${{ steps.engine-modules.outputs.modules }}${{ 
steps.cv2-modules.outputs.modules }}'
           modules=${modules: 1}
           pl_modules=`python 
tools/update_modules_check/update_modules_check.py replace $modules`
+          # remove deleted modules
+          delete_modules='${{ steps.deleted-modules.outputs.modules }}'
+          if [[ "zz"$delete_modules != "zz" ]];then
+            pl_modules=`python 
tools/update_modules_check/update_modules_check.py rm $pl_modules 
$delete_modules`
+          fi
           ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout 
-pl $pl_modules > /tmp/sub_module.txt
           sub_modules=`python 
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
           tree_modules="$modules$sub_modules"
@@ -216,6 +232,11 @@ jobs:
           modules='${{ steps.cv2-e2e-modules.outputs.modules }}${{ 
steps.cv2-flink-e2e-modules.outputs.modules }}${{ 
steps.cv2-spark-e2e-modules.outputs.modules }}${{ 
steps.engine-e2e-modules.outputs.modules }}${{ 
steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'
           modules=${modules: 1}
           pl_modules=`python 
tools/update_modules_check/update_modules_check.py replace $modules`
+          # remove deleted modules
+          delete_modules='${{ steps.deleted-modules.outputs.modules }}'
+          if [[ "zz"$delete_modules != "zz" ]];then
+            pl_modules=`python 
tools/update_modules_check/update_modules_check.py rm $pl_modules 
$delete_modules`
+          fi
           ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout 
-pl $pl_modules > /tmp/sub_module.txt
           sub_modules=`python 
tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
           tree_modules="$modules$sub_modules"
diff --git a/tools/update_modules_check/update_modules_check.py 
b/tools/update_modules_check/update_modules_check.py
index dec5dc275..bd6f11da7 100644
--- a/tools/update_modules_check/update_modules_check.py
+++ b/tools/update_modules_check/update_modules_check.py
@@ -110,6 +110,32 @@ def get_final_ut_modules(file):
     output = output[1:len(output)]
     print(output)
 
+def remove_deleted_modules(pl_modules, deleted_modules):
+    pl_modules_arr = pl_modules.replace(":", "").split(",")
+    deleted_modules_arr = deleted_modules.split(",")
+    output = ""
+    for module in pl_modules_arr:
+        if deleted_modules_arr.count(module) == 0:
+            output = output + ",:" + module
+
+    output = output[1:len(output)]
+    print(output)
+
+def get_deleted_modules(files):
+    update_files = json.loads(files)
+    modules_name_set = set([])
+    for file in update_files:
+        names = file.split('/')
+        module_name = names[len(names) - 2]
+        modules_name_set.add(module_name)
+    output_module = ""
+    if len(modules_name_set) > 0:
+        for module in modules_name_set:
+            output_module = output_module + "," + module
+
+    output_module = output_module[1:len(output_module)]
+    print(output_module)
+
 def main(argv):
     if argv[1] == "cv2":
         get_cv2_modules(argv[2])
@@ -133,6 +159,11 @@ def main(argv):
         replace_comma_to_commacolon(argv[2])
     elif argv[1] == "sub":
         get_sub_modules(argv[2])
+    elif argv[1] == "delete":
+        get_deleted_modules(argv[2])
+    elif argv[1] == "rm":
+        remove_deleted_modules(argv[2], argv[3])
+
 
 if __name__ == "__main__":
     main(sys.argv)