This is an automated email from the ASF dual-hosted git repository.
rusackas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new db8ab20f90c ci(e2e): cache a pg_dump of the loaded examples for the
Playwright jobs (#45601)
db8ab20f90c is described below
commit db8ab20f90cf929d249da39d50fc09a44bf5f727
Author: Evan Rusackas <[email protected]>
AuthorDate: Fri Oct 23 22:26:52 2026 +0100
ci(e2e): cache a pg_dump of the loaded examples for the Playwright jobs
(#45601)
Co-authored-by: Claude Fable 5.1 <[email protected]>
---
.github/workflows/bashlib.sh | 47 ++++++++++++++++++++++++++++++++++----
.github/workflows/superset-e2e.yml | 44 +++++++++++++++++++++++++++++++++++
2 files changed, 86 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/bashlib.sh b/.github/workflows/bashlib.sh
index d576f80f611..aeb241aaf13 100644
--- a/.github/workflows/bashlib.sh
+++ b/.github/workflows/bashlib.sh
@@ -123,13 +123,50 @@ testdata() {
playwright_testdata() {
cd "$GITHUB_WORKSPACE"
say "::group::Load all examples for Playwright tests"
- # must specify PYTHONPATH to make `tests.superset_test_config` importable
+
export PYTHONPATH="$GITHUB_WORKSPACE"
uv pip install --system -e .
- superset db upgrade
- superset load_test_users
- superset load_examples
- superset init
+
+ # Loading the examples takes 5+ minutes per job and every Playwright job
+ # loads the same data, so the workflow keeps a pg_dump of the result in the
+ # Actions cache (the "Restore cached examples dump" step, actions/cache).
+ # That step restores the dump here on a hit and saves whatever is here
+ # after the job on a miss. pg_dump/pg_restore run inside the Postgres
+ # service container so the client always matches the server;
+ # POSTGRES_CONTAINER is `job.services.postgres.id`, set on the step.
+ local dump_dir="/tmp/superset-playwright-testdata"
+ local dump="$dump_dir/superset.dump"
+ mkdir -p "$dump_dir"
+
+ if [[ -f "$dump" && -n "${POSTGRES_CONTAINER:-}" ]]; then
+ say "Restoring the examples database from the cached dump"
+ # The target database already has its public schema, so skip that entry
+ # of the archive's table of contents; everything else restores cleanly
+ # into the fresh database.
+ docker exec -i "$POSTGRES_CONTAINER" sh -c '
+ cat > /tmp/superset.dump &&
+ pg_restore -l /tmp/superset.dump | grep -v " SCHEMA - public " >
/tmp/superset.list &&
+ pg_restore -U superset -d superset --no-owner --no-privileges \
+ -L /tmp/superset.list /tmp/superset.dump
+ ' < "$dump"
+ # Both are no-ops on a dump built from the same migrations and views, and
+ # keep the restored database correct if the key ever lags the code.
+ superset db upgrade
+ superset init
+ else
+ superset db upgrade
+ superset load_test_users
+ superset load_examples
+ superset init
+ if [[ -n "${POSTGRES_CONTAINER:-}" ]]; then
+ say "Saving a dump of the examples database for later runs"
+ # The sqllab_test_db and admin_database schemas are created empty by
+ # setup-postgres before this runs, so leave them out of the dump.
+ docker exec "$POSTGRES_CONTAINER" \
+ pg_dump -U superset -Fc -N sqllab_test_db -N admin_database superset >
"$dump"
+ fi
+ fi
+
# Enable DML on the examples database so Playwright tests can create/drop
# temporary tables via SQL Lab without depending on external data sources.
superset shell <<'PYEOF'
diff --git a/.github/workflows/superset-e2e.yml
b/.github/workflows/superset-e2e.yml
index c22f68c1a2c..9fba43ec445 100644
--- a/.github/workflows/superset-e2e.yml
+++ b/.github/workflows/superset-e2e.yml
@@ -119,6 +119,26 @@ jobs:
uses: ./.github/actions/cached-dependencies # zizmor:
ignore[self-repository] - $/ cannot resolve an action that lives in a
submodule; ./ is required here
with:
run: setup-postgres
+ # A pg_dump of the loaded examples database. Keyed on everything that
+ # shapes the loaded rows: every backend Python file (loaders, importers,
+ # models), the example configs, migrations, the dependency pins, the CI
+ # config, and this workflow (it pins the Postgres image). The post-job
+ # save only happens on a miss; `playwright_testdata` in bashlib.sh
+ # restores the dump on a hit and writes it on a miss.
+ # hashFiles digests contents only, so renaming or removing an example
+ # file would otherwise leave the key unchanged.
+ - name: Fingerprint example file paths
+ id: examples-paths
+ run: echo "hash=$(find superset/examples -type f | LC_ALL=C sort |
sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
+ - name: Restore cached examples dump
+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: /tmp/superset-playwright-testdata
+ key: >-
+ playwright-testdata-${{ steps.examples-paths.outputs.hash }}-${{
hashFiles('superset/**/*.py', '!superset/**/__pycache__/**',
+ 'superset/examples/**', 'superset/migrations/**',
'requirements/*.txt',
+ 'pyproject.toml',
'tests/integration_tests/superset_test_config.py',
+ '.github/workflows/superset-e2e.yml',
'.github/workflows/bashlib.sh') }}
- name: Import test data
# cached-dependencies is a submodule (not a plain directory), and
# the $/ self-repository syntax resolves action files directly from
@@ -127,6 +147,8 @@ jobs:
# on the workspace-relative ./ form, consistent with every other
# workflow in the repo that references this action.
uses: ./.github/actions/cached-dependencies # zizmor:
ignore[self-repository] - $/ cannot resolve an action that lives in a
submodule; ./ is required here
+ env:
+ POSTGRES_CONTAINER: ${{ job.services.postgres.id }}
with:
run: playwright_testdata
- name: Setup Node.js
@@ -351,6 +373,26 @@ jobs:
uses: ./.github/actions/cached-dependencies # zizmor:
ignore[self-repository] - $/ cannot resolve an action that lives in a
submodule; ./ is required here
with:
run: setup-postgres
+ # A pg_dump of the loaded examples database. Keyed on everything that
+ # shapes the loaded rows: every backend Python file (loaders, importers,
+ # models), the example configs, migrations, the dependency pins, the CI
+ # config, and this workflow (it pins the Postgres image). The post-job
+ # save only happens on a miss; `playwright_testdata` in bashlib.sh
+ # restores the dump on a hit and writes it on a miss.
+ # hashFiles digests contents only, so renaming or removing an example
+ # file would otherwise leave the key unchanged.
+ - name: Fingerprint example file paths
+ id: examples-paths
+ run: echo "hash=$(find superset/examples -type f | LC_ALL=C sort |
sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
+ - name: Restore cached examples dump
+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: /tmp/superset-playwright-testdata
+ key: >-
+ playwright-testdata-${{ steps.examples-paths.outputs.hash }}-${{
hashFiles('superset/**/*.py', '!superset/**/__pycache__/**',
+ 'superset/examples/**', 'superset/migrations/**',
'requirements/*.txt',
+ 'pyproject.toml',
'tests/integration_tests/superset_test_config.py',
+ '.github/workflows/superset-e2e.yml',
'.github/workflows/bashlib.sh') }}
- name: Import test data
# cached-dependencies is a submodule (not a plain directory), and
# the $/ self-repository syntax resolves action files directly from
@@ -359,6 +401,8 @@ jobs:
# on the workspace-relative ./ form, consistent with every other
# workflow in the repo that references this action.
uses: ./.github/actions/cached-dependencies # zizmor:
ignore[self-repository] - $/ cannot resolve an action that lives in a
submodule; ./ is required here
+ env:
+ POSTGRES_CONTAINER: ${{ job.services.postgres.id }}
with:
run: playwright_testdata
- name: Setup Node.js
