This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-5207-767219abe30851cd5754dd10b93d467c9c90e4ec in repository https://gitbox.apache.org/repos/asf/texera.git
commit 9b850e2ed8927a28cf4842d8534d4cb3f0ab01b7 Author: Yicong Huang <[email protected]> AuthorDate: Mon May 25 15:49:33 2026 -0700 ci: enable Codecov Test Analytics across all test jobs (#5207) ### What changes were proposed in this PR? Enable Codecov Test Analytics for every stack we already upload coverage from. Each stack now emits JUnit-XML alongside its existing coverage report, and a second `codecov/codecov-action` step per job uploads it with `report_type: test_results`. - ScalaTest: `ThisBuild / Test / testOptions += -u target/test-reports` in the root `build.sbt`. - Vitest unit + browser configs: `reporters: ["default", ["junit", { outputFile: ... }]]`. - pytest (`python` job + `amber-integration` Python step): `--junit-xml=...`. - `bun test` (`agent-service`): `--reporter=junit --reporter-outfile=junit.xml`. Test Analytics ingestion is independent of coverage and shares the same `CODECOV_TOKEN`. Once landed, Codecov will start posting failing-test stack traces on PRs, flagging tests that have gone flaky on `main`, and tracking per-test runtime / failure-rate over time. ### Any related issues, documentation, discussions? Closes #5206. ### How was this PR tested? Ran `sbt scalafmtCheckAll` and `prettier --check` on the two modified `vitest.*.config.ts` files locally; both pass. ### Was this PR authored or co-authored using generative AI tooling? Generated-by: Claude Opus 4.7 --- .github/workflows/build.yml | 102 ++++++++++++++++++++++++++++++++++++-- build.sbt | 7 +++ frontend/vitest.browser.config.ts | 5 ++ frontend/vitest.config.ts | 5 ++ 4 files changed, 116 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8b0a5ea522..4fd056c7a7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -133,10 +133,36 @@ jobs: files: ./frontend/coverage/**/lcov.info flags: frontend fail_ci_if_error: false + - name: Upload frontend unit test results to Codecov + # vitest.config.ts adds a `junit` reporter that writes to junit.xml + # in the working dir; the @angular/build:unit-test runner forwards + # vitest config through unchanged. + if: matrix.os == 'ubuntu-latest' && !cancelled() + uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./frontend/junit.xml + flags: frontend + report_type: test_results + disable_search: true + fail_ci_if_error: false - name: Install Playwright Chromium run: yarn --cwd frontend playwright install ${{ matrix.os == 'ubuntu-latest' && '--with-deps' || '' }} chromium - name: Run frontend browser-mode tests run: yarn --cwd frontend ng run gui:test-browser + - name: Upload frontend browser-mode test results to Codecov + # vitest.browser.config.ts emits junit-browser.xml (distinct from + # the unit-test report). Same `frontend` flag — Codecov merges + # multi-file uploads under one flag. + if: matrix.os == 'ubuntu-latest' && !cancelled() + uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./frontend/junit-browser.xml + flags: frontend + report_type: test_results + disable_search: true + fail_ci_if_error: false amber: # The amber job runs the cross-cutting Scala lints (scalafmtCheckAll, @@ -255,6 +281,20 @@ jobs: files: ./**/target/scala-2.13/jacoco/report/jacoco.xml flags: amber fail_ci_if_error: false + - name: Upload amber and common test results to Codecov + # ScalaTest writes one JUnit-XML per spec under each module's + # target/test-reports/ (configured ThisBuild in build.sbt). Glob + # picks them up from every module that ran in the jacoco invocation + # above. `!cancelled()` so test failures still upload (the point + # of Test Analytics). + if: ${{ !cancelled() }} + uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./**/target/test-reports/*.xml + flags: amber + report_type: test_results + fail_ci_if_error: false amber-integration: # Runs Scala tests tagged @org.apache.texera.amber.tags.IntegrationTest — @@ -448,8 +488,21 @@ jobs: "scalafixAll --check" \ "WorkflowExecutionService/test" - name: Run Python integration tests + # --junit-xml feeds the Test Analytics upload below. run: | - cd amber && pytest -m integration -sv + cd amber && pytest -m integration --junit-xml=junit-integration.xml -sv + - name: Upload amber integration test results to Codecov + # Two separate uploads because the ScalaTest and pytest runs each + # produce their own JUnit-XMLs and Codecov keys uploads by flag. + # `!cancelled()` so test failures still upload. + if: ${{ !cancelled() }} + uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./**/target/test-reports/*.xml,./amber/junit-integration.xml + flags: amber-integration + report_type: test_results + fail_ci_if_error: false platform: # Per-service build, test, and license check for the non-amber Scala @@ -545,6 +598,17 @@ jobs: files: ./${{ matrix.service }}/target/scala-2.13/jacoco/report/jacoco.xml flags: ${{ matrix.service }} fail_ci_if_error: false + - name: Upload ${{ matrix.service }} test results to Codecov + # Per-service Test Analytics, mirroring the coverage upload flag. + if: ${{ !cancelled() }} + uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./${{ matrix.service }}/target/test-reports/*.xml + flags: ${{ matrix.service }} + report_type: test_results + disable_search: true + fail_ci_if_error: false python: if: ${{ inputs.run_python }} @@ -622,8 +686,11 @@ jobs: python -m pip install uv if [ -f amber/dev-requirements.txt ]; then uv pip install --system -r amber/dev-requirements.txt; fi - name: Test with pytest + # --junit-xml emits a JUnit-XML report alongside the coverage XML + # so the Test Analytics upload below can feed Codecov's failing- + # test PR comments and flaky-test detection on main. run: | - cd amber && pytest -m "not integration" --cov=src/main/python --cov-report=xml -sv + cd amber && pytest -m "not integration" --cov=src/main/python --cov-report=xml --junit-xml=junit.xml -sv - name: Upload python coverage to Codecov if: matrix.python-version == '3.12' && always() uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 @@ -632,6 +699,20 @@ jobs: files: ./amber/coverage.xml flags: python fail_ci_if_error: false + - name: Upload python test results to Codecov + # Test Analytics ingestion. Runs on the same 3.12 leg that uploads + # coverage to keep one canonical source per flag. `!cancelled()` + # rather than `always()` so we still upload on test failure (the + # whole point of Test Analytics) but skip cancelled runs. + if: matrix.python-version == '3.12' && !cancelled() + uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./amber/junit.xml + flags: python + report_type: test_results + disable_search: true + fail_ci_if_error: false python-state-materialization-mac: # Diagnostic leg: cross-region state materialization is reported to @@ -713,7 +794,10 @@ jobs: - name: Typecheck run: bun run typecheck - name: Run unit tests - run: bun test --coverage --coverage-reporter=lcov + # --reporter=junit emits a JUnit-XML alongside coverage so the + # Test Analytics upload below can feed Codecov's failing-test PR + # comments and flaky-test detection on main. + run: bun test --coverage --coverage-reporter=lcov --reporter=junit --reporter-outfile=junit.xml - name: Upload agent-service coverage to Codecov if: matrix.os == 'ubuntu-latest' && always() uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 @@ -722,3 +806,15 @@ jobs: files: ./agent-service/coverage/lcov.info flags: agent-service fail_ci_if_error: false + - name: Upload agent-service test results to Codecov + # Test Analytics ingestion. Runs on the same ubuntu leg that + # uploads coverage. `!cancelled()` so test failures still upload. + if: matrix.os == 'ubuntu-latest' && !cancelled() + uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./agent-service/junit.xml + flags: agent-service + report_type: test_results + disable_search: true + fail_ci_if_error: false diff --git a/build.sbt b/build.sbt index b7b6b3cfb2..8268fc7a11 100644 --- a/build.sbt +++ b/build.sbt @@ -24,6 +24,13 @@ import com.typesafe.sbt.packager.universal.UniversalPlugin.autoImport.Universal ThisBuild / Test / javaOptions ++= JdkOptions.jvmFlags((ThisBuild / baseDirectory).value) +// Emit one JUnit-XML file per spec under each module's target/test-reports/. +// Codecov Test Analytics ingests these via `report_type: test_results` to +// surface failing-test stack traces in PR comments and flag tests that have +// gone flaky on main. ScalaTest's `-u` argument is additive — module-level +// testOptions (e.g. amber/build.sbt's filter args) continue to apply. +ThisBuild / Test / testOptions += Tests.Argument(TestFrameworks.ScalaTest, "-u", "target/test-reports") + // sbt-jacoco emits only HTML by default; add XML so Codecov can consume // per-module jacoco.xml at target/scala-2.13/jacoco/report/jacoco.xml. // JacocoPlugin defines a project-scoped default that overrides ThisBuild, diff --git a/frontend/vitest.browser.config.ts b/frontend/vitest.browser.config.ts index b24f875a4f..eee0624ff1 100644 --- a/frontend/vitest.browser.config.ts +++ b/frontend/vitest.browser.config.ts @@ -42,6 +42,11 @@ export default defineConfig({ ], }, test: { + // Emit a JUnit-XML report alongside the default console reporter so + // Codecov Test Analytics can ingest browser-mode failures and detect + // flakies on main. Written to a distinct filename so the upload step + // can disambiguate it from the unit-test report. + reporters: ["default", ["junit", { outputFile: "junit-browser.xml" }]], globals: true, setupFiles: ["src/test-zone-setup.ts"], browser: { diff --git a/frontend/vitest.config.ts b/frontend/vitest.config.ts index 5462d42cf4..82a35b7afe 100644 --- a/frontend/vitest.config.ts +++ b/frontend/vitest.config.ts @@ -21,6 +21,11 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { + // Emit a JUnit-XML report alongside the default console reporter so + // Codecov Test Analytics can ingest failing-test stack traces and + // detect flakies on main. `default` stays first so CI logs read the + // same as before. + reporters: ["default", ["junit", { outputFile: "junit.xml" }]], // Make describe/it/expect/vi/beforeEach/etc available as globals so // existing Jasmine-style specs don't need a per-file import sweep. // Paired with `vitest/globals` triple-slash in src/vitest-globals.d.ts.
