This is an automated email from the ASF dual-hosted git repository.
ash pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 7417f81 Run UI tests selectively when UI files have changed (#15009)
7417f81 is described below
commit 7417f81d36ad02c2a9d7feb9b9f881610f50ceba
Author: Ash Berlin-Taylor <ash_git...@firemirror.com>
AuthorDate: Wed Mar 31 23:10:00 2021 +0100
Run UI tests selectively when UI files have changed (#15009)
If we only change the new React UI files, we don't want to have to run
the whole python test suite!
---
.github/workflows/ci.yml | 37 ++++++++++++++++++++++++++++++
.pre-commit-config.yaml | 7 ++++++
BREEZE.rst | 2 +-
STATIC_CODE_CHECKS.rst | 2 ++
breeze-complete | 1 +
scripts/ci/selective_ci_checks.sh | 46 +++++++++++++++++++++++++++++++++++---
scripts/ci/static_checks/eslint.sh | 31 +++++++++++++++++++++++++
7 files changed, 122 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 49fb2e7..3f90b43 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -147,6 +147,7 @@ jobs:
sqliteExclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
kubernetesExclude: ${{ steps.selective-checks.outputs.kubernetes-exclude
}}
run-tests: ${{ steps.selective-checks.outputs.run-tests }}
+ run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }}
run-kubernetes-tests: ${{
steps.selective-checks.outputs.run-kubernetes-tests }}
basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only
}}
image-build: ${{ steps.selective-checks.outputs.image-build }}
@@ -344,6 +345,12 @@
pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-v
key:
"pre-commit-no-pylint-${{steps.host-python-version.outputs.host-python-version}}-\
${{ hashFiles('.pre-commit-config.yaml') }}"
restore-keys:
pre-commit-no-pylint-${{steps.host-python-version.outputs.host-python-version}}
+
+ - name: "Cache eslint"
+ uses: actions/cache@v2
+ with:
+ path: 'airflow/ui/node_modules'
+ key: ${{ runner.os }}-ui-node-modules-${{
hashFiles('airflow/ui/**/yarn.lock') }}
- name: "Static checks: except pylint"
run: ./scripts/ci/static_checks/run_static_checks.sh
env:
@@ -1264,3 +1271,33 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tags: true
force: true
branch: master
+
+ tests-ui:
+ timeout-minutes: 10
+ name: React UI tests
+ runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ needs: [build-info, ci-images]
+ env:
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
+ if: needs.build-info.outputs.run-ui-tests == 'true'
+ steps:
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v2
+ with:
+ persist-credentials: false
+ submodules: recursive
+ - name: "Setup node"
+ uses: actions/setup-node@v2
+ with:
+ node-version: 14
+ - name: "Free space"
+ run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{
env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
+ run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
+ - name: "Cache eslint"
+ uses: actions/cache@v2
+ with:
+ path: 'airflow/ui/node_modules'
+ key: ${{ runner.os }}-ui-node-modules-${{
hashFiles('airflow/ui/**/yarn.lock') }}
+ - run: yarn --cwd airflow/ui/ install --frozen-lockfile --non-interactive
+ - run: yarn --cwd airflow/ui/ run test
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index e3cc75a..50e18ed 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -620,6 +620,13 @@ repos:
entry: "./scripts/ci/pre_commit/pre_commit_flake8.sh"
files: \.py$
pass_filenames: true
+ - id: ui-lint
+ name: ESLint against airflow/ui
+ language: node
+ 'types_or': [javascript, tsx, ts]
+ files: ^airflow/ui/
+ entry: "scripts/ci/static_checks/eslint.sh"
+ pass_filenames: false
- id: bats-in-container-tests
name: Run in container bats tests
language: system
diff --git a/BREEZE.rst b/BREEZE.rst
index 55cc46a..2d0a346 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -2298,7 +2298,7 @@ This is the current syntax for `./breeze <./breeze>`_:
pre-commit-hook-names provide-create-sessions
providers-init-file provider-yamls
pydevd pydocstyle pylint pylint-tests python-no-log-warn
pyupgrade
restrict-start_date rst-backticks setup-order
setup-extra-packages shellcheck
- sort-in-the-wild sort-spelling-wordlist stylelint
trailing-whitespace
+ sort-in-the-wild sort-spelling-wordlist stylelint
trailing-whitespace ui-lint
update-breeze-file update-extras update-local-yml-file
update-setup-cfg-file
version-sync yamllint
diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst
index 69bc52a..1481c2f 100644
--- a/STATIC_CODE_CHECKS.rst
+++ b/STATIC_CODE_CHECKS.rst
@@ -170,6 +170,8 @@ require Breeze Docker images to be installed locally:
-----------------------------------
---------------------------------------------------------------- ------------
``trailing-whitespace`` Removes trailing whitespace at end of
line
-----------------------------------
---------------------------------------------------------------- ------------
+``ui-lint`` Static checks of airflow/ui/ folder
+-----------------------------------
---------------------------------------------------------------- ------------
``update-breeze-file`` Update output of breeze command in
BREEZE.rst
-----------------------------------
---------------------------------------------------------------- ------------
``update-extras`` Updates extras in the documentation
diff --git a/breeze-complete b/breeze-complete
index ba84987..af1f2e4 100644
--- a/breeze-complete
+++ b/breeze-complete
@@ -137,6 +137,7 @@ sort-in-the-wild
sort-spelling-wordlist
stylelint
trailing-whitespace
+ui-lint
update-breeze-file
update-extras
update-local-yml-file
diff --git a/scripts/ci/selective_ci_checks.sh
b/scripts/ci/selective_ci_checks.sh
index 0b40e60..c2adf2d 100755
--- a/scripts/ci/selective_ci_checks.sh
+++ b/scripts/ci/selective_ci_checks.sh
@@ -45,7 +45,7 @@ fi
function check_upgrade_to_newer_dependencies_needed() {
# shellcheck disable=SC2153
if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ||
- ${GITHUB_EVENT_NAME} == 'push' || ${GITHUB_EVENT_NAME} ==
"scheduled" ]]; then
+ ${GITHUB_EVENT_NAME=} == 'push' || ${GITHUB_EVENT_NAME=} ==
"scheduled" ]]; then
# Trigger upgrading to latest constraints where label is set or when
# SHA of the merge commit triggers rebuilding layer in the docker image
# Each build that upgrades to latest constraints will get truly latest
constraints, not those
@@ -203,6 +203,10 @@ function set_upgrade_to_newer_dependencies() {
initialization::ga_output upgrade-to-newer-dependencies "${@}"
}
+function needs_ui_tests() {
+ initialization::ga_output run-ui-tests "${@}"
+}
+
if [[ ${DEFAULT_BRANCH} == "master" ]]; then
ALL_TESTS="Always API Core Other CLI Providers WWW Integration"
else
@@ -224,6 +228,7 @@ function set_outputs_run_everything_and_exit() {
set_docs_build "true"
set_image_build "true"
set_upgrade_to_newer_dependencies "${upgrade_to_newer_dependencies}"
+ needs_ui_tests "true"
exit
}
@@ -233,6 +238,7 @@ function set_outputs_run_all_tests() {
set_test_types "${ALL_TESTS}"
set_basic_checks_only "false"
set_image_build "true"
+ needs_ui_tests "true"
kubernetes_tests_needed="true"
}
@@ -249,6 +255,7 @@ function set_output_skip_all_tests_and_docs_and_exit() {
set_docs_build "false"
set_image_build "false"
set_upgrade_to_newer_dependencies "false"
+ needs_ui_tests "false"
exit
}
@@ -265,6 +272,7 @@ function set_output_skip_tests_but_build_images_and_exit() {
set_docs_build "true"
set_image_build "true"
set_upgrade_to_newer_dependencies "${upgrade_to_newer_dependencies}"
+ needs_ui_tests "false"
exit
}
@@ -338,7 +346,7 @@ function check_if_setup_files_changed() {
function check_if_javascript_security_scans_should_be_run() {
start_end::group_start "Check Javascript security scans"
local pattern_array=(
- "^airflow/.*\.js"
+ "^airflow/.*\.[jt]sx?"
"^airflow/.*\.lock"
)
show_changed_files
@@ -416,6 +424,24 @@ function check_if_docs_should_be_generated() {
start_end::group_end
}
+function check_if_ui_tests_should_be_run() {
+ start_end::group_start "Check UI"
+ local pattern_array=(
+ "^airflow/ui/.*\.[tj]sx?$"
+ # tsconfig.json, package.json, etc.
+ "^airflow/ui/[^/]+\.json$"
+ "^airflow/ui/.*\.lock$"
+ )
+ show_changed_files
+
+ if [[ $(count_changed_files) == "0" ]]; then
+ needs_ui_tests "false"
+ else
+ needs_ui_tests "true"
+ fi
+ start_end::group_end
+}
+
ANY_PY_FILES_CHANGED=(
"\.py$"
@@ -548,6 +574,18 @@ function get_count_www_files() {
start_end::group_end
}
+function get_count_ui_files() {
+ start_end::group_start "Count ui files"
+ local pattern_array=(
+ "^airflow/ui/"
+ )
+ show_changed_files
+ COUNT_UI_CHANGED_FILES=$(count_changed_files)
+ echo "Files count: ${COUNT_UI_CHANGED_FILES}"
+ readonly COUNT_UI_CHANGED_FILES
+ start_end::group_end
+}
+
function get_count_kubernetes_files() {
start_end::group_start "Count kubernetes files"
local pattern_array=(
@@ -565,7 +603,7 @@ function get_count_kubernetes_files() {
function calculate_test_types_to_run() {
start_end::group_start "Count core/other files"
- COUNT_CORE_OTHER_CHANGED_FILES=$((COUNT_ALL_CHANGED_FILES -
COUNT_WWW_CHANGED_FILES - COUNT_PROVIDERS_CHANGED_FILES -
COUNT_CLI_CHANGED_FILES - COUNT_API_CHANGED_FILES -
COUNT_KUBERNETES_CHANGED_FILES))
+ COUNT_CORE_OTHER_CHANGED_FILES=$((COUNT_ALL_CHANGED_FILES -
COUNT_WWW_CHANGED_FILES - COUNT_UI_CHANGED_FILES -
COUNT_PROVIDERS_CHANGED_FILES - COUNT_CLI_CHANGED_FILES -
COUNT_API_CHANGED_FILES - COUNT_KUBERNETES_CHANGED_FILES))
readonly COUNT_CORE_OTHER_CHANGED_FILES
echo
@@ -664,12 +702,14 @@ check_if_api_tests_should_be_run
check_if_api_codegen_should_be_run
check_if_javascript_security_scans_should_be_run
check_if_python_security_scans_should_be_run
+check_if_ui_tests_should_be_run
check_if_tests_are_needed_at_all
get_count_all_files
get_count_api_files
get_count_cli_files
get_count_providers_files
get_count_www_files
+get_count_ui_files
get_count_kubernetes_files
calculate_test_types_to_run
diff --git a/scripts/ci/static_checks/eslint.sh
b/scripts/ci/static_checks/eslint.sh
new file mode 100755
index 0000000..3722e54
--- /dev/null
+++ b/scripts/ci/static_checks/eslint.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+MOUNT_ALL_LOCAL_SOURCES="false"
+
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+
+build_images::prepare_ci_build
+
+build_images::rebuild_ci_image_if_needed
+
+docker run "${EXTRA_DOCKER_FLAGS[@]}" \
+ --entrypoint "/bin/bash" \
+ "${AIRFLOW_CI_IMAGE}" \
+ -c 'cd airflow/ui && yarn --frozen-lockfile --non-interactive && yarn run
lint "${@}"' "${@#airflow/ui/}"
