This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new e80d194b8f fix: improve change detection for GHAs (#27904)
e80d194b8f is described below
commit e80d194b8f211d4064972b6a9377feb2911fe6d1
Author: Maxime Beauchemin <[email protected]>
AuthorDate: Mon Apr 8 16:20:25 2024 -0700
fix: improve change detection for GHAs (#27904)
---
.github/actions/change-detector/action.yml | 31 ++++
.github/workflows/codeql-analysis.yml | 11 +-
.github/workflows/docker.yml | 21 ++-
.github/workflows/no-op.yml | 114 ---------------
.github/workflows/superset-cli.yml | 18 ++-
.github/workflows/superset-e2e.yml | 27 ++--
.github/workflows/superset-frontend.yml | 36 ++---
.../workflows/superset-python-integrationtest.yml | 60 ++++----
.github/workflows/superset-python-misc.yml | 25 ++--
.github/workflows/superset-python-presto-hive.yml | 54 ++++---
.github/workflows/superset-python-unittest.yml | 26 +---
.github/workflows/superset-translations.yml | 20 ++-
.pre-commit-config.yaml | 1 +
scripts/change_detector.py | 156 +++++++++++++++++++++
scripts/ci_check_no_file_changes.sh | 68 ---------
superset/__init__.py | 1 +
16 files changed, 338 insertions(+), 331 deletions(-)
diff --git a/.github/actions/change-detector/action.yml
b/.github/actions/change-detector/action.yml
new file mode 100644
index 0000000000..d0f356e771
--- /dev/null
+++ b/.github/actions/change-detector/action.yml
@@ -0,0 +1,31 @@
+name: 'Change Detector'
+description: 'Detects file changes for pull request and push events'
+inputs:
+ token:
+ description: 'GitHub token for authentication'
+ required: true
+outputs:
+ python:
+ description: 'Whether Python-related files were changed'
+ value: ${{ steps.change-detector.outputs.python }}
+ frontend:
+ description: 'Whether frontend-related files were changed'
+ value: ${{ steps.change-detector.outputs.frontend }}
+ docker:
+ description: 'Whether docker-related files were changed'
+ value: ${{ steps.change-detector.outputs.docker }}
+ docs:
+ description: 'Whether docs-related files were changed'
+ value: ${{ steps.change-detector.outputs.docs }}
+runs:
+ using: 'composite'
+ steps:
+ - name: Detect file changes
+ id: change-detector
+ run: |
+ python --version
+ python scripts/change_detector.py
+ shell: bash
+ env:
+ GITHUB_TOKEN: ${{ inputs.token }}
+ GITHUB_OUTPUT: ${{ github.output }}
diff --git a/.github/workflows/codeql-analysis.yml
b/.github/workflows/codeql-analysis.yml
index c200f9cb36..15395bbfde 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -3,13 +3,9 @@ name: "CodeQL"
on:
push:
branches: ["master", "[0-9].[0-9]"]
- paths:
- - "superset/**"
pull_request:
# The branches below must be a subset of the branches above
branches: ["master"]
- paths:
- - "superset/**"
schedule:
- cron: "0 4 * * *"
@@ -37,6 +33,12 @@ jobs:
- name: Checkout repository
uses: actions/checkout@v4
+ - name: Check for file changes
+ id: check
+ uses: ./.github/actions/change-detector/
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
+
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
@@ -50,6 +52,7 @@ jobs:
# queries: security-extended,security-and-quality
- name: Perform CodeQL Analysis
+ if: steps.check.outputs.python || steps.check.outputs.frontend
uses: github/codeql-action/analyze@v3
with:
category: "/language:${{matrix.language}}"
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index cb9d579780..18001241ab 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -39,28 +39,39 @@ jobs:
steps:
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+
+ - name: Check for file changes
+ id: check
+ uses: ./.github/actions/change-detector/
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
+
- name: Set up QEMU
+ if: steps.check.outputs.python || steps.check.outputs.frontend ||
steps.check.outputs.docker
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
+ if: steps.check.outputs.python || steps.check.outputs.frontend ||
steps.check.outputs.docker
uses: docker/setup-buildx-action@v3
- name: Try to login to DockerHub
+ if: steps.check.outputs.python || steps.check.outputs.frontend ||
steps.check.outputs.docker
continue-on-error: true
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v4
- with:
- persist-credentials: false
-
- name: Setup supersetbot
+ if: steps.check.outputs.python || steps.check.outputs.frontend ||
steps.check.outputs.docker
uses: ./.github/actions/setup-supersetbot/
- name: Build Docker Image
+ if: steps.check.outputs.python || steps.check.outputs.frontend ||
steps.check.outputs.docker
shell: bash
run: |
# Single platform builds in pull_request context to speed things up
diff --git a/.github/workflows/no-op.yml b/.github/workflows/no-op.yml
deleted file mode 100644
index 9cc9e8bec3..0000000000
--- a/.github/workflows/no-op.yml
+++ /dev/null
@@ -1,114 +0,0 @@
-# no-op.yml
-#
-# Purpose:
-# This workflow provides a workaround for the "required status checks" feature
in GitHub Actions
-# when using path-specific conditions in other workflows. Required checks
might remain in a "Pending"
-# state if the conditions are not met, thus blocking pull requests from being
merged.
-# This no-op (no operation) workflow provides dummy success statuses for these
required jobs when
-# the real jobs do not run due to path-specific conditions.
-#
-# How it works:
-# - It defines jobs with the same names as the required jobs in the main
workflows.
-# - These jobs simply execute a command (`exit 0`) to succeed immediately.
-# - When a pull request is created or updated, both this no-op workflow and
the main workflows are triggered.
-# - If the main workflows' jobs don't run (due to path conditions), these
no-op jobs provide successful statuses.
-# - If the main workflows' jobs do run and fail, their failure statuses take
precedence,
-# ensuring that pull requests are not merged with failing checks.
-#
-# Usage:
-# - Ensure that the job names in this workflow match exactly the names of the
corresponding jobs in the main workflows.
-# - This workflow should be kept as-is, without path-specific conditions.
-
-name: no-op Checks
-on: pull_request
-
-jobs:
- frontend-build:
- runs-on: ubuntu-latest
- steps:
- - name: No-op for frontend-build
- run: |
- echo "This is a no-op step for frontend-build to ensure a successful
status."
- exit 0
-
- pre-commit:
- strategy:
- matrix:
- python-version: ["3.9"]
- runs-on: ubuntu-latest
- steps:
- - name: No-op for pre-commit
- run: |
- echo "This is a no-op step for pre-commit to ensure a successful
status."
- exit 0
-
- python-lint:
- strategy:
- matrix:
- python-version: ["3.9", "3.10"]
- runs-on: ubuntu-latest
- steps:
- - name: No-op for python-lint
- run: |
- echo "This is a no-op step for python-lint to ensure a successful
status."
- exit 0
- test-postgres-hive:
- strategy:
- matrix:
- python-version: ["3.9", "3.10"]
- runs-on: ubuntu-latest
- steps:
- - name: No-op for frontend-build
- run: |
- echo "This is a no-op step for test-postgres-postgres to ensure a
successful status when skipped."
- exit 0
- test-postgres-presto:
- strategy:
- matrix:
- python-version: ["3.9", "3.10"]
- runs-on: ubuntu-latest
- steps:
- - name: No-op for frontend-build
- run: |
- echo "This is a no-op step for test-postgres-postgres to ensure a
successful status when skipped."
- exit 0
- unit-tests:
- strategy:
- matrix:
- python-version: ["3.9", "3.10"]
- runs-on: ubuntu-latest
- steps:
- - name: No-op for frontend-build
- run: |
- echo "This is a no-op step for unit-tests to ensure a successful
status when skipped."
- exit 0
- test-mysql:
- strategy:
- matrix:
- python-version: ["3.9"]
- runs-on: ubuntu-latest
- steps:
- - name: No-op for test-mysql
- run: |
- echo "This is a no-op step for test-mysql to ensure a successful
status when skipped."
- exit 0
- test-postgres:
- strategy:
- matrix:
- python-version: ["3.9"]
- runs-on: ubuntu-latest
- steps:
- - name: No-op for test-postgres
- run: |
- echo "This is a no-op step for test-postgres to ensure a successful
status when skipped."
- exit 0
- test-sqlite:
- strategy:
- matrix:
- python-version: ["3.9"]
- runs-on: ubuntu-latest
- steps:
- - name: No-op for test-sqlite
- run: |
- echo "This is a no-op step for test-sqlite to ensure a successful
status when skipped."
- exit 0
diff --git a/.github/workflows/superset-cli.yml
b/.github/workflows/superset-cli.yml
index f0e037197e..1d9870a0b2 100644
--- a/.github/workflows/superset-cli.yml
+++ b/.github/workflows/superset-cli.yml
@@ -44,29 +44,27 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- - name: Check if python changes are present
+ - name: Check for file changes
id: check
- env:
- GITHUB_REPO: ${{ github.repository }}
- PR_NUMBER: ${{ github.event.pull_request.number }}
- continue-on-error: true
- run: ./scripts/ci_check_no_file_changes.sh python
+ uses: ./.github/actions/change-detector/
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
+ if: steps.check.outputs.python
uses: ./.github/actions/setup-backend/
- if: steps.check.outcome == 'failure'
- name: Setup Postgres
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
uses: ./.github/actions/cached-dependencies
with:
run: setup-postgres
- name: superset init
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: |
pip install -e .
superset db upgrade
superset load_test_users
- name: superset load_examples
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: |
# load examples without test data
superset load_examples --load-big-data
diff --git a/.github/workflows/superset-e2e.yml
b/.github/workflows/superset-e2e.yml
index 9086c6d0eb..231643d73b 100644
--- a/.github/workflows/superset-e2e.yml
+++ b/.github/workflows/superset-e2e.yml
@@ -60,47 +60,46 @@ jobs:
ref: "refs/pull/${{ github.event.number }}/merge"
persist-credentials: false
submodules: recursive
- - name: Check if python or frontend changes are present
+ - name: Check for file changes
id: check
- env:
- GITHUB_REPO: ${{ github.repository }}
- PR_NUMBER: ${{ github.event.pull_request.number }}
- continue-on-error: true
- run: ./scripts/ci_check_no_file_changes.sh python frontend
+ uses: ./.github/actions/change-detector/
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python || steps.check.outputs.frontend
- name: Setup postgres
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python || steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: setup-postgres
- name: Import test data
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python || steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: testdata
- name: Setup Node.js
+ if: steps.check.outputs.python || steps.check.outputs.frontend
uses: actions/setup-node@v4
with:
node-version: "16"
- name: Install npm dependencies
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python || steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: npm-install
- name: Build javascript packages
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python || steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: build-instrumented-assets
- name: Install cypress
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python || steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: cypress-install
- name: Run Cypress
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python || steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
env:
CYPRESS_BROWSER: ${{ matrix.browser }}
@@ -109,7 +108,7 @@ jobs:
run: cypress-run-all
- name: Upload Artifacts
uses: actions/upload-artifact@v4
- if: failure()
+ if: steps.check.outputs.python || steps.check.outputs.frontend
with:
name: screenshots
path: ${{ github.workspace
}}/superset-frontend/cypress-base/cypress/screenshots
diff --git a/.github/workflows/superset-frontend.yml
b/.github/workflows/superset-frontend.yml
index 2b078d485b..20add61f7e 100644
--- a/.github/workflows/superset-frontend.yml
+++ b/.github/workflows/superset-frontend.yml
@@ -5,12 +5,8 @@ on:
branches:
- "master"
- "[0-9].[0-9]"
- paths:
- - "superset-frontend/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
- paths:
- - "superset-frontend/**"
# cancel previous workflow jobs for PRs
concurrency:
@@ -28,62 +24,60 @@ jobs:
submodules: recursive
- name: Check npm lock file version
run: ./scripts/ci_check_npm_lock_version.sh
./superset-frontend/package-lock.json
- - name: Check if frontend changes are present
+ - name: Check for file changes
id: check
- env:
- GITHUB_REPO: ${{ github.repository }}
- PR_NUMBER: ${{ github.event.pull_request.number }}
- continue-on-error: true
- run: ./scripts/ci_check_no_file_changes.sh frontend
+ uses: ./.github/actions/change-detector/
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Node.js
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.frontend
uses: actions/setup-node@v4
with:
node-version: "18"
- name: Install dependencies
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: npm-install
- name: eslint
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run eslint -- . --quiet
- name: tsc
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run type
- name: prettier
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run prettier-check
- name: Build plugins packages
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: npm run plugins:build
- name: Build plugins Storybook
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: npm run plugins:build-storybook
- name: superset-ui/core coverage
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run core:cover
- name: unit tests
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run test -- --coverage --silent
# todo: remove this step when fix generator as a project in root
jest.config.js
- name: generator-superset unit tests
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.frontend
working-directory: ./superset-frontend/packages/generator-superset
run: npx jest
- name: Upload code coverage
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: ../.github/workflows/codecov.sh -c -F javascript
diff --git a/.github/workflows/superset-python-integrationtest.yml
b/.github/workflows/superset-python-integrationtest.yml
index 75eea5c70a..4d710d5bb5 100644
--- a/.github/workflows/superset-python-integrationtest.yml
+++ b/.github/workflows/superset-python-integrationtest.yml
@@ -43,33 +43,31 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- - name: Check if python changes are present
+ - name: Check for file changes
id: check
- env:
- GITHUB_REPO: ${{ github.repository }}
- PR_NUMBER: ${{ github.event.pull_request.number }}
- continue-on-error: true
- run: ./scripts/ci_check_no_file_changes.sh python
+ uses: ./.github/actions/change-detector/
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
with:
python-version: ${{ matrix.python-version }}
- name: Setup MySQL
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
uses: ./.github/actions/cached-dependencies
with:
run: |
setup-mysql
- name: Run celery
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python integration tests (MySQL)
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: |
./scripts/python_tests.sh
- name: Upload code coverage
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: |
bash .github/workflows/codecov.sh -c -F python -F mysql
test-postgres:
@@ -102,33 +100,31 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- - name: Check if python changes are present
+ - name: Check for file changes
id: check
- env:
- GITHUB_REPO: ${{ github.repository }}
- PR_NUMBER: ${{ github.event.pull_request.number }}
- continue-on-error: true
- run: ./scripts/ci_check_no_file_changes.sh python
+ uses: ./.github/actions/change-detector/
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
with:
python-version: ${{ matrix.python-version }}
- name: Setup Postgres
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
uses: ./.github/actions/cached-dependencies
with:
run: |
setup-postgres
- name: Run celery
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python integration tests (PostgreSQL)
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: |
./scripts/python_tests.sh
- name: Upload code coverage
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: |
bash .github/workflows/codecov.sh -c -F python -F postgres
@@ -156,33 +152,31 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- - name: Check if python changes are present
+ - name: Check for file changes
id: check
- env:
- GITHUB_REPO: ${{ github.repository }}
- PR_NUMBER: ${{ github.event.pull_request.number }}
- continue-on-error: true
- run: ./scripts/ci_check_no_file_changes.sh python
+ uses: ./.github/actions/change-detector/
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
uses: ./.github/actions/cached-dependencies
with:
run: |
# sqlite needs this working directory
mkdir ${{ github.workspace }}/.temp
- name: Run celery
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python integration tests (SQLite)
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: |
./scripts/python_tests.sh
- name: Upload code coverage
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: |
bash .github/workflows/codecov.sh -c -F python -F sqlite
diff --git a/.github/workflows/superset-python-misc.yml
b/.github/workflows/superset-python-misc.yml
index 8eb34d939a..2aacf8799b 100644
--- a/.github/workflows/superset-python-misc.yml
+++ b/.github/workflows/superset-python-misc.yml
@@ -6,12 +6,8 @@ on:
branches:
- "master"
- "[0-9].[0-9]"
- paths:
- - "superset/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
- paths:
- - "superset/**"
# cancel previous workflow jobs for PRs
concurrency:
@@ -30,20 +26,18 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- - name: Check if python changes are present
+ - name: Check for file changes
id: check
- env:
- GITHUB_REPO: ${{ github.repository }}
- PR_NUMBER: ${{ github.event.pull_request.number }}
- continue-on-error: true
- run: ./scripts/ci_check_no_file_changes.sh python
+ uses: ./.github/actions/change-detector/
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
with:
python-version: ${{ matrix.python-version }}
- name: pylint
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
# `-j 0` run Pylint in parallel
run: pylint -j 0 superset
@@ -58,9 +52,16 @@ jobs:
with:
persist-credentials: false
submodules: recursive
+ - name: Check for file changes
+ id: check
+ uses: ./.github/actions/change-detector/
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
+ if: steps.check.outputs.python
uses: ./.github/actions/setup-backend/
with:
python-version: ${{ matrix.python-version }}
- name: Test babel extraction
+ if: steps.check.outputs.python
run: flask fab babel-extract --target superset/translations --output
superset/translations/messages.pot --config superset/translations/babel.cfg -k
_,__,t,tn,tct
diff --git a/.github/workflows/superset-python-presto-hive.yml
b/.github/workflows/superset-python-presto-hive.yml
index dd8d6612aa..66c6952c7d 100644
--- a/.github/workflows/superset-python-presto-hive.yml
+++ b/.github/workflows/superset-python-presto-hive.yml
@@ -6,12 +6,8 @@ on:
branches:
- "master"
- "[0-9].[0-9]"
- paths:
- - "superset/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
- paths:
- - "superset/**"
# cancel previous workflow jobs for PRs
concurrency:
@@ -59,30 +55,30 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- - name: Check if python changes are present
+ - name: Check for file changes
id: check
- env:
- GITHUB_REPO: ${{ github.repository }}
- PR_NUMBER: ${{ github.event.pull_request.number }}
- continue-on-error: true
- run: ./scripts/ci_check_no_file_changes.sh python
+ uses: ./.github/actions/change-detector/
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python == 'true'
- name: Setup Postgres
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
uses: ./.github/actions/cached-dependencies
with:
- run: setup-postgres
+ run: |
+ echo "${{ steps.check.outputs.python }}"
+ setup-postgres
- name: Run celery
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: |
./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
- name: Upload code coverage
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: |
bash .github/workflows/codecov.sh -c -F python -F presto
@@ -118,38 +114,36 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- - name: Check if python changes are present
+ - name: Check for file changes
id: check
- env:
- GITHUB_REPO: ${{ github.repository }}
- PR_NUMBER: ${{ github.event.pull_request.number }}
- continue-on-error: true
- run: ./scripts/ci_check_no_file_changes.sh python
+ uses: ./.github/actions/change-detector/
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
- name: Create csv upload directory
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: sudo mkdir -p /tmp/.superset/uploads
- name: Give write access to the csv upload directory
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: sudo chown -R $USER:$USER /tmp/.superset
- name: Start hadoop and hive
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: docker compose -f scripts/databases/hive/docker-compose.yml up -d
- name: Setup Python
uses: ./.github/actions/setup-backend/
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
- name: Setup Postgres
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
uses: ./.github/actions/cached-dependencies
with:
run: setup-postgres
- name: Run celery
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: |
./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
- name: Upload code coverage
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: |
bash .github/workflows/codecov.sh -c -F python -F hive
diff --git a/.github/workflows/superset-python-unittest.yml
b/.github/workflows/superset-python-unittest.yml
index c5bf8bf46d..48e2c21c26 100644
--- a/.github/workflows/superset-python-unittest.yml
+++ b/.github/workflows/superset-python-unittest.yml
@@ -6,18 +6,8 @@ on:
branches:
- "master"
- "[0-9].[0-9]"
- paths:
- - "superset/**"
- - "requirements/**"
- - "tests/unit_tests/**"
- - "scripts/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
- paths:
- - "superset/**"
- - "requirements/**"
- - "tests/unit_tests/**"
- - "scripts/**"
# cancel previous workflow jobs for PRs
concurrency:
@@ -38,26 +28,24 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- - name: Check if python changes are present
+ - name: Check for file changes
id: check
- env:
- GITHUB_REPO: ${{ github.repository }}
- PR_NUMBER: ${{ github.event.pull_request.number }}
- continue-on-error: true
- run: ./scripts/ci_check_no_file_changes.sh python
+ uses: ./.github/actions/change-detector/
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
with:
python-version: ${{ matrix.python-version }}
- name: Python unit tests
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
env:
SUPERSET_TESTENV: true
SUPERSET_SECRET_KEY: not-a-secret
run: |
pytest --durations-min=0.5 --cov-report= --cov=superset
./tests/common ./tests/unit_tests --cache-clear
- name: Upload code coverage
- if: steps.check.outcome == 'failure'
+ if: steps.check.outputs.python
run: |
bash .github/workflows/codecov.sh -c -F python -F unit
diff --git a/.github/workflows/superset-translations.yml
b/.github/workflows/superset-translations.yml
index 9a8b8df1e3..734e9bf752 100644
--- a/.github/workflows/superset-translations.yml
+++ b/.github/workflows/superset-translations.yml
@@ -14,7 +14,7 @@ concurrency:
cancel-in-progress: true
jobs:
- frontend-check:
+ frontend-check-translations:
runs-on: ubuntu-20.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -22,15 +22,25 @@ jobs:
with:
persist-credentials: false
submodules: recursive
+
+ - name: Check for file changes
+ id: check
+ uses: ./.github/actions/change-detector/
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
+
- name: Setup Node.js
+ if: steps.check.outputs.frontend
uses: actions/setup-node@v4
with:
node-version: '18'
- name: Install dependencies
+ if: steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: npm-install
- name: lint
+ if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run check-translation
@@ -46,9 +56,17 @@ jobs:
with:
persist-credentials: false
submodules: recursive
+ - name: Check for file changes
+ id: check
+ uses: ./.github/actions/change-detector/
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
+
- name: Setup Python
+ if: steps.check.outputs.python
uses: ./.github/actions/setup-backend/
with:
python-version: ${{ matrix.python-version }}
- name: Test babel extraction
+ if: steps.check.outputs.python
run: ./scripts/babel_update.sh
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 3549b3f473..a476dc3ab1 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -23,6 +23,7 @@ repos:
rev: v3.4.0
hooks:
- id: pyupgrade
+ exclude: scripts/change_detector.py
args:
- --py39-plus
- repo: https://github.com/hadialqattan/pycln
diff --git a/scripts/change_detector.py b/scripts/change_detector.py
new file mode 100755
index 0000000000..df2c872a4a
--- /dev/null
+++ b/scripts/change_detector.py
@@ -0,0 +1,156 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import json
+import os
+import re
+import subprocess
+from typing import List
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+# Define patterns for each group of files you're interested in
+PATTERNS = {
+ "python": [
+ r"^\.github/workflows/.*python",
+ r"^tests/",
+ r"^superset/",
+ r"^scripts/",
+ r"^setup\.py",
+ r"^requirements/.+\.txt",
+ r"^.pylintrc",
+ ],
+ "frontend": [
+ r"^\.github/workflows/.*(bashlib|frontend|e2e)",
+ r"^superset-frontend/",
+ ],
+ "docker": [
+ r"^Dockerfile$",
+ r"^docker/",
+ ],
+ "docs": [
+ r"^docs/",
+ ],
+}
+GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN")
+
+
+def fetch_files_github_api(url: str): # type: ignore
+ """Fetches data using GitHub API."""
+ req = Request(url)
+ req.add_header("Authorization", f"token {GITHUB_TOKEN}")
+ req.add_header("Accept", "application/vnd.github.v3+json")
+
+ print(f"Fetching from {url}")
+ with urlopen(req) as response:
+ body = response.read()
+ return json.loads(body)
+
+
+def fetch_changed_files_pr(repo: str, pr_number: str) -> List[str]:
+ """Fetches files changed in a PR using the GitHub API."""
+ url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}/files";
+ files = fetch_files_github_api(url)
+ return [file_info["filename"] for file_info in files]
+
+
+def fetch_changed_files_push(repo: str, sha: str) -> List[str]:
+ """Fetches files changed in the last commit for push events using GitHub
API."""
+ # Fetch commit details to get the parent SHA
+ commit_url = f"https://api.github.com/repos/{repo}/commits/{sha}";
+ commit_data = fetch_files_github_api(commit_url)
+ if "parents" not in commit_data or len(commit_data["parents"]) < 1:
+ raise RuntimeError("No parent commit found for comparison.")
+ parent_sha = commit_data["parents"][0]["sha"]
+ # Compare the current commit against its parent
+ compare_url =
f"https://api.github.com/repos/{repo}/compare/{parent_sha}...{sha}";
+ comparison_data = fetch_files_github_api(compare_url)
+ return [file["filename"] for file in comparison_data["files"]]
+
+
+def detect_changes(files: List[str], check_patterns: List) -> bool: # type:
ignore
+ """Detects if any of the specified files match the provided patterns."""
+ for file in files:
+ for pattern in check_patterns:
+ if re.match(pattern, file):
+ return True
+ return False
+
+
+def print_files(files: List[str]) -> None:
+ print("\n".join([f"- {s}" for s in files]))
+
+
+def main(event_type: str, sha: str, repo: str) -> None:
+ """Main function to check for file changes based on event context."""
+ print("SHA:", sha)
+ if event_type == "pull_request":
+ pr_number = os.getenv("GITHUB_REF", "").split("/")[-2]
+ files = fetch_changed_files_pr(repo, pr_number)
+ print(f"PR files:")
+ print_files(files)
+
+ elif event_type == "push":
+ files = fetch_changed_files_push(repo, sha)
+ print(f"Files touched since previous commit:")
+ print_files(files)
+ else:
+ raise ValueError("Unsupported event type")
+
+ changes_detected = {}
+ for group, regex_patterns in PATTERNS.items():
+ patterns_compiled = [re.compile(p) for p in regex_patterns]
+ changes_detected[group] = detect_changes(files, patterns_compiled)
+
+ # Output results
+ output_path = os.getenv("GITHUB_OUTPUT") or "/tmp/GITHUB_OUTPUT.txt"
+ with open(output_path, "a") as f:
+ for check, changed in changes_detected.items():
+ if changed:
+ print(f"{check}={str(changed).lower()}", file=f)
+ print(f"Triggering group: {check}")
+
+
+def get_git_sha() -> str:
+ return os.getenv("GITHUB_SHA") or subprocess.check_output(
+ ["git", "rev-parse", "HEAD"]
+ ).strip().decode("utf-8")
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(
+ description="Detect file changes based on event context"
+ )
+ parser.add_argument(
+ "--event-type",
+ default=os.getenv("GITHUB_EVENT_NAME") or "push",
+ help="The type of event that triggered the workflow",
+ )
+ parser.add_argument(
+ "--sha",
+ default=get_git_sha(),
+ help="The commit SHA for push events or PR head SHA",
+ )
+ parser.add_argument(
+ "--repo",
+ default=os.getenv("GITHUB_REPOSITORY") or "apache/superset",
+ help="GitHub repository in the format owner/repo",
+ )
+ args = parser.parse_args()
+
+ main(args.event_type, args.sha, args.repo)
diff --git a/scripts/ci_check_no_file_changes.sh
b/scripts/ci_check_no_file_changes.sh
deleted file mode 100755
index d4851ea668..0000000000
--- a/scripts/ci_check_no_file_changes.sh
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Usage instructions:
-#
-# to check for python changes, run with CHECKS=python
-# To check for frontend changes, run with CHECKS=frontend
-# To check for python and frontend changes, run with CHECKS="python frontend"
-if [[ -z ${PR_NUMBER} ]]; then
- echo "Not a PR; Exiting with FAILURE code"
- exit 1
-fi
-
-URL="https://api.github.com/repos/${GITHUB_REPO}/pulls/${PR_NUMBER}/files?per_page=1000";
-FILES=$(curl -s -X GET -G "${URL}" | jq -r '.[] | .filename')
-
-REGEXES=()
-for CHECK in "$@"
-do
- if [[ ${CHECK} == "python" ]]; then
-
REGEX="(^\.github\/workflows\/.*python|^tests\/|^superset\/|^scripts\/|^setup\.py|^requirements\/.+\.txt|^\.pylintrc)"
- echo "Searching for changes in python files"
- elif [[ ${CHECK} == "frontend" ]]; then
-
REGEX="(^\.github\/workflows\/.*(bashlib|frontend|e2e)|^superset-frontend\/)"
- echo "Searching for changes in frontend files"
- else
- echo "Invalid check: \"${CHECK}\". Falling back to exiting with FAILURE
code"
- exit 1
- fi
- REGEXES=("${REGEXES[@]}" "${REGEX}")
-done
-echo
-
-cat<<EOF
-CHANGED FILES:
-$FILES
-
-EOF
-
-for FILE in ${FILES}
-do
- for REGEX in "${REGEXES[@]}"
- do
- if [[ "${FILE}" =~ ${REGEX} ]]; then
- echo "Detected changes in following file: ${FILE}"
- echo "Exiting with FAILURE code"
- exit 1
- fi
- done
-done
-echo "No changes detected... Exiting with SUCCESS code"
-exit 0
diff --git a/superset/__init__.py b/superset/__init__.py
index 5c8ff3ca2d..d96697de99 100644
--- a/superset/__init__.py
+++ b/superset/__init__.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
"""Package's main module!"""
+
from flask import current_app, Flask
from werkzeug.local import LocalProxy
