This is an automated email from the ASF dual-hosted git repository.
driazati pushed a commit to branch driazati/test_lint_split
in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 58aa95af1042a15585ede4dd9e9fc6914aeefefe
Author: driazati <driaz...@users.noreply.github.com>
AuthorDate: Tue Oct 4 00:19:10 2022 -0700
[ci] Split lint into steps
This splits lints into parallel steps based on the job rather than
arbitrary shards.
---
Jenkinsfile | 200 +++++++++++++++++++++++++++++++++++---
ci/jenkins/Lint.groovy.j2 | 73 ++++++++++++--
ci/jenkins/macros.j2 | 14 +--
tests/scripts/task_lint.sh | 95 ------------------
tests/scripts/task_short_lints.sh | 47 +++++++++
5 files changed, 306 insertions(+), 123 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index e964ac79a3..bb31fe324e 100755
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -45,7 +45,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2022-09-26T10:48:49.577077
+// Generated at 2022-10-04T00:20:36.200805
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// NOTE: these lines are scanned by docker/dev_common.sh. Please update the
regex as needed. -->
@@ -610,19 +610,17 @@ def build_docker_images() {
def lint() {
stage('Lint') {
parallel(
- 'Lint 1 of 2': {
- node('CPU-SMALL') {
+ 'other lints': {
+ node('CPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") {
init_git()
docker_init(ci_lint)
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
- 'TVM_NUM_SHARDS=2',
- 'TEST_STEP_NAME=Lint',
- 'TVM_SHARD_INDEX=0',
+ 'TEST_STEP_NAME=other lints',
"SKIP_SLOW_TESTS=${skip_slow_tests}"], {
sh (
- script: "${docker_run} ${ci_lint}
./tests/scripts/task_lint.sh",
+ script: "${docker_run} ${ci_lint}
./tests/scripts/task_short_lints.sh",
label: 'Run lint',
)
})
@@ -630,16 +628,14 @@ def lint() {
}
}
},
- 'Lint 2 of 2': {
- node('CPU-SMALL') {
+ 'pylint': {
+ node('CPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") {
init_git()
docker_init(ci_lint)
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
- 'TVM_NUM_SHARDS=2',
- 'TEST_STEP_NAME=Lint',
- 'TVM_SHARD_INDEX=1',
+ 'TEST_STEP_NAME=pylint',
"SKIP_SLOW_TESTS=${skip_slow_tests}"], {
sh (
script: "${docker_run} ${ci_lint}
./tests/scripts/task_lint.sh",
@@ -649,6 +645,186 @@ def lint() {
}
}
}
+ },
+ 'black': {
+ node('CPU') {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") {
+ init_git()
+ docker_init(ci_lint)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'TEST_STEP_NAME=black',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh (
+ script: 'tests/lint/git-black.sh',
+ label: 'black check',
+ )
+ })
+ }
+ }
+ }
+ },
+ 'flake8': {
+ node('CPU') {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") {
+ init_git()
+ docker_init(ci_lint)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'TEST_STEP_NAME=flake8',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh (
+ script: 'tests/lint/flake8.sh',
+ label: 'Linting the Python code with flake8',
+ )
+ })
+ }
+ }
+ }
+ },
+ 'mypy': {
+ node('CPU') {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") {
+ init_git()
+ docker_init(ci_lint)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'TEST_STEP_NAME=mypy',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh (
+ script: 'tests/scripts/task_mypy.sh',
+ label: 'Type checking with MyPy ',
+ )
+ })
+ }
+ }
+ }
+ },
+ 'jni': {
+ node('CPU') {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") {
+ init_git()
+ docker_init(ci_lint)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'TEST_STEP_NAME=jni',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh (
+ script: 'tests/lint/jnilint.sh',
+ label: 'Linting the JNI code',
+ )
+ })
+ }
+ }
+ }
+ },
+ 'pylint': {
+ node('CPU') {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") {
+ init_git()
+ docker_init(ci_lint)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'TEST_STEP_NAME=pylint',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh (
+ script: 'tests/lint/pylint.sh',
+ label: 'Linting the Python code with pylint',
+ )
+ })
+ }
+ }
+ }
+ },
+ 'cppdocs': {
+ node('CPU') {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") {
+ init_git()
+ docker_init(ci_lint)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'TEST_STEP_NAME=cppdocs',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh (
+ script: 'tests/lint/cppdocs.sh',
+ label: 'Checking C++ documentation',
+ )
+ })
+ }
+ }
+ }
+ },
+ 'asf headers': {
+ node('CPU') {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") {
+ init_git()
+ docker_init(ci_lint)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'TEST_STEP_NAME=asf headers',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh (
+ script: 'tests/lint/check_asf_header.sh --local',
+ label: 'Checking ASF license headers',
+ )
+ })
+ }
+ }
+ }
+ },
+ 'cpplint': {
+ node('CPU') {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") {
+ init_git()
+ docker_init(ci_lint)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'TEST_STEP_NAME=cpplint',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh (
+ script: 'tests/lint/cpplint.sh',
+ label: 'Linting the C++ code',
+ )
+ })
+ }
+ }
+ }
+ },
+ 'clang-format': {
+ node('CPU') {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") {
+ init_git()
+ docker_init(ci_lint)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'TEST_STEP_NAME=clang-format',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh (
+ script: 'tests/lint/git-clang-format.sh',
+ label: 'clang-format check',
+ )
+ })
+ }
+ }
+ }
+ },
+ 'rust': {
+ node('CPU') {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") {
+ init_git()
+ docker_init(ci_lint)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'TEST_STEP_NAME=rust',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh (
+ script: 'tests/lint/rust_format.sh',
+ label: 'Rust check',
+ )
+ })
+ }
+ }
+ }
},
)
}
diff --git a/ci/jenkins/Lint.groovy.j2 b/ci/jenkins/Lint.groovy.j2
index 3ede64301c..13f111f622 100644
--- a/ci/jenkins/Lint.groovy.j2
+++ b/ci/jenkins/Lint.groovy.j2
@@ -1,19 +1,78 @@
def lint() {
stage('Lint') {
parallel(
- {% call m.sharded_lint_step(
- name='Lint',
- num_shards=2,
- node='CPU-SMALL',
- ws='tvm/lint',
- docker_image='ci_lint',
+ {% call m.lint_step(name='other lints') %}
+ sh (
+ script: "${docker_run} ${ci_lint}
./tests/scripts/task_short_lints.sh",
+ label: 'Run lint',
)
- %}
+ {% endcall %}
+ {% call m.lint_step(name='pylint') %}
sh (
script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh",
label: 'Run lint',
)
{% endcall %}
+ {% call m.lint_step('black') %}
+ sh (
+ script: 'tests/lint/git-black.sh',
+ label: 'black check',
+ )
+ {% endcall %}
+ {% call m.lint_step('flake8') %}
+ sh (
+ script: 'tests/lint/flake8.sh',
+ label: 'Linting the Python code with flake8',
+ )
+ {% endcall %}
+ {% call m.lint_step('mypy') %}
+ sh (
+ script: 'tests/scripts/task_mypy.sh',
+ label: 'Type checking with MyPy ',
+ )
+ {% endcall %}
+ {% call m.lint_step('jni') %}
+ sh (
+ script: 'tests/lint/jnilint.sh',
+ label: 'Linting the JNI code',
+ )
+ {% endcall %}
+ {% call m.lint_step('pylint') %}
+ sh (
+ script: 'tests/lint/pylint.sh',
+ label: 'Linting the Python code with pylint',
+ )
+ {% endcall %}
+ {% call m.lint_step('cppdocs') %}
+ sh (
+ script: 'tests/lint/cppdocs.sh',
+ label: 'Checking C++ documentation',
+ )
+ {% endcall %}
+ {% call m.lint_step('asf headers') %}
+ sh (
+ script: 'tests/lint/check_asf_header.sh --local',
+ label: 'Checking ASF license headers',
+ )
+ {% endcall %}
+ {% call m.lint_step('cpplint') %}
+ sh (
+ script: 'tests/lint/cpplint.sh',
+ label: 'Linting the C++ code',
+ )
+ {% endcall %}
+ {% call m.lint_step('clang-format') %}
+ sh (
+ script: 'tests/lint/git-clang-format.sh',
+ label: 'clang-format check',
+ )
+ {% endcall %}
+ {% call m.lint_step('rust') %}
+ sh (
+ script: 'tests/lint/rust_format.sh',
+ label: 'Rust check',
+ )
+ {% endcall %}
)
}
}
diff --git a/ci/jenkins/macros.j2 b/ci/jenkins/macros.j2
index e6e69097b0..572bc5d9e5 100644
--- a/ci/jenkins/macros.j2
+++ b/ci/jenkins/macros.j2
@@ -66,18 +66,15 @@ def {{ method_name }}() {
{% endfor %}
{% endmacro %}
-{% macro sharded_lint_step(name, num_shards, docker_image, node, ws) %}
-{% for shard_index in range(1, num_shards + 1) %}
- '{{ name }} {{ shard_index }} of {{ num_shards }}': {
- node('{{ node }}') {
- ws({{ per_exec_ws(ws) }}) {
+{% macro lint_step(name) %}
+ '{{ name }}': {
+ node('CPU') {
+ ws({{ per_exec_ws('tvm/lint') }}) {
init_git()
- docker_init({{ docker_image }})
+ docker_init(ci_lint)
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
- 'TVM_NUM_SHARDS={{ num_shards }}',
'TEST_STEP_NAME={{ name }}',
- 'TVM_SHARD_INDEX={{ shard_index - 1 }}',
"SKIP_SLOW_TESTS=${skip_slow_tests}"], {
{{ caller() | trim | indent(width=6) }}
})
@@ -85,7 +82,6 @@ def {{ method_name }}() {
}
}
},
-{% endfor %}
{% endmacro %}
{% macro test_step_body(name, node, ws, docker_image, platform) %}
diff --git a/tests/scripts/task_lint.sh b/tests/scripts/task_lint.sh
deleted file mode 100755
index 84f4652337..0000000000
--- a/tests/scripts/task_lint.sh
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -euxo pipefail
-
-cleanup()
-{
- rm -rf /tmp/$$.*
-}
-trap cleanup 0
-
-
-# These shards are solely for CI to enable the lint job to have some
parallelism.
-
-function shard1 {
- echo "Convert scripts to Python..."
- tests/scripts/task_convert_scripts_to_python.sh
-
- echo "Check Jenkinsfile generation"
- python3 ci/jenkins/generate.py --check
-
- echo "Checking file types..."
- python3 tests/lint/check_file_type.py
-
- echo "Checking CMake <-> LibInfo options mirroring"
- python3 tests/lint/check_cmake_options.py
-
- echo "Checking that all sphinx-gallery docs override urllib.request.Request"
- python3 tests/lint/check_request_hook.py
-
- echo "black check..."
- tests/lint/git-black.sh
-
- echo "Linting the Python code with flake8..."
- tests/lint/flake8.sh
-
- echo "Type checking with MyPy ..."
- tests/scripts/task_mypy.sh
-
- echo "Checking for non-inclusive language with blocklint..."
- tests/lint/blocklint.sh
-
- echo "Linting the JNI code..."
- tests/lint/jnilint.sh
-}
-
-function shard2 {
- echo "Linting the Python code with pylint..."
- tests/lint/pylint.sh
-
- echo "Checking C++ documentation..."
- tests/lint/cppdocs.sh
-
- echo "Checking ASF license headers..."
- tests/lint/check_asf_header.sh --local
-
- echo "Linting the C++ code..."
- tests/lint/cpplint.sh
-
- echo "clang-format check..."
- tests/lint/git-clang-format.sh
-
- echo "Rust check..."
- tests/lint/rust_format.sh
-
- echo "Docker check..."
- tests/lint/docker-format.sh
-}
-
-
-if [[ -n ${TVM_SHARD_INDEX+x} ]]; then
- if [[ "$TVM_SHARD_INDEX" == "0" ]]; then
- shard1
- else
- shard2
- fi
-else
- shard1
- shard2
-fi
diff --git a/tests/scripts/task_short_lints.sh
b/tests/scripts/task_short_lints.sh
new file mode 100755
index 0000000000..6f244a5c72
--- /dev/null
+++ b/tests/scripts/task_short_lints.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euxo pipefail
+
+cleanup()
+{
+ rm -rf /tmp/$$.*
+}
+trap cleanup 0
+
+
+echo "Convert scripts to Python..."
+tests/scripts/task_convert_scripts_to_python.sh
+
+echo "Check Jenkinsfile generation"
+python3 ci/jenkins/generate.py --check
+
+echo "Checking file types..."
+python3 tests/lint/check_file_type.py
+
+echo "Checking CMake <-> LibInfo options mirroring"
+python3 tests/lint/check_cmake_options.py
+
+echo "Checking that all sphinx-gallery docs override urllib.request.Request"
+python3 tests/lint/check_request_hook.py
+
+echo "Docker check..."
+tests/lint/docker-format.sh
+
+echo "Checking for non-inclusive language with blocklint..."
+tests/lint/blocklint.sh
