This is an automated email from the ASF dual-hosted git repository. imbajin pushed a commit to branch goal-test in repository https://gitbox.apache.org/repos/asf/hugegraph-ai.git
commit 6de7e26a322bebbfa04a803cb580533aa139985e Author: imbajin <[email protected]> AuthorDate: Sun May 31 02:21:08 2026 +0800 ci(quality): split test gates and add coverage ratchet - separate unit, contract, integration, and smoke CI paths - publish baseline coverage artifacts and ratchet commands - keep external provider tests outside default PR gates --- .github/workflows/hugegraph-llm.yml | 111 +++++++++++++++++++-- .github/workflows/hugegraph-python-client.yml | 60 +++++++++-- .../checkpoints/07-coverage-ratchet.md | 77 ++++++++++++++ .../coverage/combined-baseline.json | 1 + .workflow/quality-program/quality-state.json | 25 ++++- docs/quality/coverage-ratchet.md | 61 +++++++++++ .../2026-05-31-hugegraph-ai-quality-program.md | 14 +-- 7 files changed, 318 insertions(+), 31 deletions(-) diff --git a/.github/workflows/hugegraph-llm.yml b/.github/workflows/hugegraph-llm.yml index 6b1ba56c..faa8046d 100644 --- a/.github/workflows/hugegraph-llm.yml +++ b/.github/workflows/hugegraph-llm.yml @@ -27,7 +27,55 @@ on: pull_request: jobs: - build: + llm-unit-contract: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: | + ~/.cache/uv + ~/nltk_data + key: ${{ runner.os }}-uv-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml', 'uv.lock') }} + restore-keys: | + ${{ runner.os }}-uv-${{ matrix.python-version }}- + + - name: Install dependencies + run: | + uv sync --extra llm --extra dev + uv run python -c "import nltk; nltk.download('stopwords'); nltk.download('punkt')" + + - name: Run unit tests + env: + SKIP_EXTERNAL_SERVICES: true + run: | + uv run pytest hugegraph-llm/src/tests -m "unit or contract" --cov=hugegraph_llm --cov-report=term --cov-report=xml:.workflow/quality-program/coverage/llm-unit-contract.xml -q + + - name: Upload coverage artifact + uses: actions/upload-artifact@v4 + with: + name: llm-unit-contract-coverage-${{ matrix.python-version }} + path: .workflow/quality-program/coverage/llm-unit-contract.xml + if-no-files-found: error + + llm-hugegraph-boundary: runs-on: ubuntu-latest strategy: fail-fast: false @@ -71,15 +119,62 @@ jobs: uv sync --extra llm --extra dev uv run python -c "import nltk; nltk.download('stopwords'); nltk.download('punkt')" - - name: Run unit tests - working-directory: hugegraph-llm + - name: Run HugeGraph boundary tests env: - SKIP_EXTERNAL_SERVICES: true + HUGEGRAPH_REQUIRED: true + HUGEGRAPH_URL: http://127.0.0.1:8080 + HUGEGRAPH_GRAPH: hugegraph + HUGEGRAPH_USER: admin + HUGEGRAPH_PASSWORD: admin + SKIP_EXTERNAL_SERVICES: false + run: | + uv run pytest hugegraph-llm/src/tests -m "integration and hugegraph" -v --tb=short + + llm-core-smoke: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11"] + + services: + hugegraph: + image: hugegraph/hugegraph:1.7.0 + env: + PASSWORD: admin + options: --health-cmd="curl -f http://localhost:8080/versions || exit 1" --health-interval=10s --health-timeout=5s --health-retries=8 + ports: + - 8080:8080 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv run: | - uv run pytest src/tests/config/ src/tests/document/ src/tests/middleware/ src/tests/operators/ src/tests/models/ src/tests/indices/ src/tests/test_utils.py -v --tb=short + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: | + ~/.cache/uv + ~/nltk_data + key: ${{ runner.os }}-uv-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml', 'uv.lock') }} + restore-keys: | + ${{ runner.os }}-uv-${{ matrix.python-version }}- + + - name: Install dependencies + run: | + uv sync --extra llm --extra dev + uv run python -c "import nltk; nltk.download('stopwords'); nltk.download('punkt')" - - name: Run integration tests - working-directory: hugegraph-llm + - name: Run deterministic core smoke tests env: HUGEGRAPH_REQUIRED: true HUGEGRAPH_URL: http://127.0.0.1:8080 @@ -88,4 +183,4 @@ jobs: HUGEGRAPH_PASSWORD: admin SKIP_EXTERNAL_SERVICES: false run: | - uv run pytest src/tests/integration/test_graph_rag_pipeline.py src/tests/integration/test_kg_construction.py src/tests/integration/test_rag_pipeline.py -v --tb=short + uv run pytest hugegraph-llm/src/tests/integration -m "smoke" -v --tb=short diff --git a/.github/workflows/hugegraph-python-client.yml b/.github/workflows/hugegraph-python-client.yml index 436d80d8..a6150fa6 100644 --- a/.github/workflows/hugegraph-python-client.yml +++ b/.github/workflows/hugegraph-python-client.yml @@ -8,7 +8,52 @@ on: pull_request: jobs: - build: + client-unit-contract: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + + - name: Cache dependencies + id: cache-deps + uses: actions/cache@v4 + with: + path: | + ~/.cache/uv + key: ${{ runner.os }}-uv-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-uv-${{ matrix.python-version }}- + + - name: Install dependencies with uv sync + run: | + uv sync --extra python-client --extra dev + + - name: Run unit and contract tests + run: | + uv run pytest hugegraph-python-client/src/tests -m "unit or contract" --cov=pyhugegraph --cov-report=term --cov-report=xml:.workflow/quality-program/coverage/client-unit-contract.xml -q + + - name: Upload coverage artifact + uses: actions/upload-artifact@v4 + with: + name: client-unit-contract-coverage-${{ matrix.python-version }} + path: .workflow/quality-program/coverage/client-unit-contract.xml + if-no-files-found: error + + client-hugegraph-integration: runs-on: ubuntu-latest strategy: fail-fast: false @@ -20,7 +65,7 @@ jobs: image: hugegraph/hugegraph:1.7.0 env: PASSWORD: admin - options: --health-cmd="curl -f http://localhost:8080/versions || exit 1" --health-interval=10s --health-timeout=5s --health-retries=5 + options: --health-cmd="curl -f http://localhost:8080/versions || exit 1" --health-interval=10s --health-timeout=5s --health-retries=8 ports: - 8080:8080 @@ -51,13 +96,7 @@ jobs: run: | uv sync --extra python-client --extra dev - - name: Test example - run: | - ls -al - export PYTHONPATH=$(pwd)/hugegraph-python-client/src && echo ${PYTHONPATH} - uv run python hugegraph-python-client/src/pyhugegraph/example/hugegraph_example.py - - - name: Test with pytest + - name: Run HugeGraph integration tests env: HUGEGRAPH_REQUIRED: true HUGEGRAPH_URL: http://127.0.0.1:8080 @@ -65,5 +104,4 @@ jobs: HUGEGRAPH_USER: admin HUGEGRAPH_PASSWORD: admin run: | - uv run pytest - working-directory: hugegraph-python-client + uv run pytest hugegraph-python-client/src/tests -m "integration and hugegraph" -v --tb=short diff --git a/.workflow/quality-program/checkpoints/07-coverage-ratchet.md b/.workflow/quality-program/checkpoints/07-coverage-ratchet.md new file mode 100644 index 00000000..639c17c0 --- /dev/null +++ b/.workflow/quality-program/checkpoints/07-coverage-ratchet.md @@ -0,0 +1,77 @@ +# G6 Coverage Ratchet Checkpoint + +## Status + +Complete. + +## CI Split + +Client workflow: + +| Job | Service | Command | +|---|---|---| +| `client-unit-contract` | none | `uv run pytest hugegraph-python-client/src/tests -m "unit or contract" --cov=pyhugegraph --cov-report=term --cov-report=xml:.workflow/quality-program/coverage/client-unit-contract.xml -q` | +| `client-hugegraph-integration` | `hugegraph/hugegraph:1.7.0` | `uv run pytest hugegraph-python-client/src/tests -m "integration and hugegraph" -v --tb=short` | + +LLM workflow: + +| Job | Service | Command | +|---|---|---| +| `llm-unit-contract` | none | `uv run pytest hugegraph-llm/src/tests -m "unit or contract" --cov=hugegraph_llm --cov-report=term --cov-report=xml:.workflow/quality-program/coverage/llm-unit-contract.xml -q` | +| `llm-hugegraph-boundary` | `hugegraph/hugegraph:1.7.0` | `uv run pytest hugegraph-llm/src/tests -m "integration and hugegraph" -v --tb=short` | +| `llm-core-smoke` | `hugegraph/hugegraph:1.7.0` | `uv run pytest hugegraph-llm/src/tests/integration -m "smoke" -v --tb=short` | + +External-provider tests are excluded from default PR gates. Unit/contract jobs do not start Docker. + +## Coverage Baseline + +The plan's original combined baseline command failed because both workspace packages expose `tests.conftest`, causing pytest import-path mismatch during cross-package collection. + +Passing command: + +```bash +uv run pytest --import-mode=importlib hugegraph-python-client/src/tests hugegraph-llm/src/tests -m 'unit or contract' --cov=pyhugegraph --cov=hugegraph_llm --cov-report=term --cov-report=json:.workflow/quality-program/coverage/combined-baseline.json +``` + +Result: + +```text +185 passed, 2 skipped, 193 deselected, 4 warnings in 12.30s +TOTAL 7975 statements, 3115 covered, 4860 missing, 39% coverage +``` + +## Ratchet Documentation + +Created `docs/quality/coverage-ratchet.md` with: + +- initial local ratchet areas +- layer-specific local commands +- combined baseline command with `--import-mode=importlib` +- rules for production bug fixes, HugeGraph boundaries, and external-provider behavior + +## Verification + +G6 verification commands: + +```bash +uv run python -c "import yaml, pathlib; [yaml.safe_load(pathlib.Path(p).read_text()) for p in ['.github/workflows/hugegraph-llm.yml','.github/workflows/hugegraph-python-client.yml']]" +uv run ruff format --check . +uv run ruff check . +uv run pytest hugegraph-python-client/src/tests -m "unit or contract" -q +uv run pytest hugegraph-llm/src/tests -m "unit or contract" -q +HUGEGRAPH_REQUIRED=true uv run pytest hugegraph-python-client/src/tests -m "integration and hugegraph" -v --tb=short +HUGEGRAPH_REQUIRED=true uv run pytest hugegraph-llm/src/tests -m "integration and hugegraph" -v --tb=short +uv run pytest hugegraph-llm/src/tests/integration -m "smoke" -v --tb=short +``` + +Results: + +- YAML workflow parse: passed +- `git diff --check`: passed +- `uv run ruff format --check .`: 342 files already formatted +- `uv run ruff check .`: all checks passed +- Client unit/contract: 17 passed, 59 deselected +- LLM unit/contract: 168 passed, 2 skipped, 134 deselected +- Client HugeGraph integration: 59 passed, 17 deselected +- LLM HugeGraph integration: 5 passed, 299 deselected +- LLM smoke: 4 passed, 15 deselected diff --git a/.workflow/quality-program/coverage/combined-baseline.json b/.workflow/quality-program/coverage/combined-baseline.json new file mode 100644 index 00000000..68b42e41 --- /dev/null +++ b/.workflow/quality-program/coverage/combined-baseline.json @@ -0,0 +1 @@ +{"meta": {"format": 3, "version": "7.10.4", "timestamp": "2026-05-31T02:16:06.749334", "branch_coverage": false, "show_contexts": false}, "files": {"hugegraph-llm/src/hugegraph_llm/__init__.py": {"executed_lines": [1], "summary": {"covered_lines": 0, "num_statements": 0, "percent_covered": 100.0, "percent_covered_display": "100", "missing_lines": 0, "excluded_lines": 0}, "missing_lines": [], "excluded_lines": [], "functions": {"": {"executed_lines": [1], "summary": {"covered_lines": 0, " [...] diff --git a/.workflow/quality-program/quality-state.json b/.workflow/quality-program/quality-state.json index 0a3591c0..e4bf12cb 100644 --- a/.workflow/quality-program/quality-state.json +++ b/.workflow/quality-program/quality-state.json @@ -1,5 +1,5 @@ { - "current_goal": "G6", + "current_goal": "G7", "repo_sha_start": "d048bc9fa07042833549dd1def5ff3222c47be70", "base_branch": "main", "open_pr_snapshot_time": "2026-05-30T17:18:30Z", @@ -10,7 +10,8 @@ "G2", "G3", "G4", - "G5" + "G5", + "G6" ], "files_touched": [ ".workflow/quality-program/README.md", @@ -84,7 +85,10 @@ "hugegraph-llm/src/tests/data/quality_program/text2gremlin_schema.json", "hugegraph-llm/src/tests/integration/test_core_kg_smoke.py", "hugegraph-llm/src/tests/integration/test_core_graphrag_smoke.py", - "hugegraph-llm/src/tests/integration/test_core_text2gremlin_smoke.py" + "hugegraph-llm/src/tests/integration/test_core_text2gremlin_smoke.py", + "docs/quality/coverage-ratchet.md", + ".workflow/quality-program/coverage/combined-baseline.json", + ".workflow/quality-program/checkpoints/07-coverage-ratchet.md" ], "production_changes": [ { @@ -189,7 +193,18 @@ "uv run pytest hugegraph-llm/src/tests/integration -m 'smoke and not hugegraph' -v --tb=short", "uv run ruff format --check hugegraph-llm/src/tests/integration/test_core_kg_smoke.py hugegraph-llm/src/tests/integration/test_core_graphrag_smoke.py hugegraph-llm/src/tests/integration/test_core_text2gremlin_smoke.py", "uv run ruff check hugegraph-llm/src/tests/integration/test_core_kg_smoke.py hugegraph-llm/src/tests/integration/test_core_graphrag_smoke.py hugegraph-llm/src/tests/integration/test_core_text2gremlin_smoke.py", - "HUGEGRAPH_REQUIRED=true uv run pytest hugegraph-llm/src/tests/integration -m 'smoke' -q" + "HUGEGRAPH_REQUIRED=true uv run pytest hugegraph-llm/src/tests/integration -m 'smoke' -q", + "uv run pytest hugegraph-python-client/src/tests hugegraph-llm/src/tests -m 'unit or contract' --cov=pyhugegraph --cov=hugegraph_llm --cov-report=term --cov-report=json:.workflow/quality-program/coverage/combined-baseline.json", + "uv run pytest --import-mode=importlib hugegraph-python-client/src/tests hugegraph-llm/src/tests -m 'unit or contract' --cov=pyhugegraph --cov=hugegraph_llm --cov-report=term --cov-report=json:.workflow/quality-program/coverage/combined-baseline.json", + "uv run python -c \"import yaml, pathlib; [yaml.safe_load(pathlib.Path(p).read_text()) for p in ['.github/workflows/hugegraph-llm.yml','.github/workflows/hugegraph-python-client.yml']]\"", + "git diff --check", + "uv run ruff format --check .", + "uv run ruff check .", + "uv run pytest hugegraph-python-client/src/tests -m \"unit or contract\" -q", + "uv run pytest hugegraph-llm/src/tests -m \"unit or contract\" -q", + "HUGEGRAPH_REQUIRED=true uv run pytest hugegraph-python-client/src/tests -m \"integration and hugegraph\" -v --tb=short", + "HUGEGRAPH_REQUIRED=true uv run pytest hugegraph-llm/src/tests -m \"integration and hugegraph\" -v --tb=short", + "uv run pytest hugegraph-llm/src/tests/integration -m \"smoke\" -v --tb=short" ], "known_failures": [], "deferred_items": [ @@ -199,5 +214,5 @@ "Avoid #240 vector/property embedding abstraction surface.", "Quarantine client/test fixes overlapping #323/#329/#342 until exact changed files are inspected." ], - "next_recommended_action": "Start G6 coverage ratchet and CI split" + "next_recommended_action": "Start G7 deferred refactor queue and final report" } diff --git a/docs/quality/coverage-ratchet.md b/docs/quality/coverage-ratchet.md new file mode 100644 index 00000000..17c83d0f --- /dev/null +++ b/docs/quality/coverage-ratchet.md @@ -0,0 +1,61 @@ +# Coverage Ratchet + +## Principles + +- Start with local areas, not a full-repo threshold. +- New production logic requires tests. +- Bug fixes require regression tests. +- HugeGraph boundaries need Layer B or Layer C evidence. +- Thresholds may start low but must not decrease. +- External provider, vector DB, and UI credential paths stay outside default PR ratchets. + +## Initial Ratchet Areas + +- `pyhugegraph` +- `hugegraph_llm.operators.hugegraph_op` +- `hugegraph_llm.operators.llm_op` +- `hugegraph_llm.api` +- `hugegraph_llm.api.models` + +## Baseline + +The combined unit/contract baseline is stored at: + +```text +.workflow/quality-program/coverage/combined-baseline.json +``` + +Current combined baseline: + +| Scope | Covered lines | Statements | Missing lines | Coverage | +|---|---:|---:|---:|---:| +| `pyhugegraph` + `hugegraph_llm` unit/contract | 3115 | 7975 | 4860 | 39% | + +The original combined command hits a pytest import-path collision because both workspace packages define a `tests.conftest` module. Use `--import-mode=importlib` for combined workspace baselines until the test package layout is normalized. + +## Local Commands + +```bash +uv run pytest hugegraph-python-client/src/tests -m "unit or contract" --cov=pyhugegraph --cov-report=term +uv run pytest hugegraph-llm/src/tests/operators/hugegraph_op -m "unit or contract" --cov=hugegraph_llm.operators.hugegraph_op --cov-report=term +uv run pytest hugegraph-llm/src/tests/operators/llm_op -m "unit or contract" --cov=hugegraph_llm.operators.llm_op --cov-report=term +uv run pytest hugegraph-llm/src/tests/api -m "unit or contract" --cov=hugegraph_llm.api --cov-report=term +``` + +Combined workspace baseline command: + +```bash +uv run pytest --import-mode=importlib hugegraph-python-client/src/tests hugegraph-llm/src/tests \ + -m "unit or contract" \ + --cov=pyhugegraph \ + --cov=hugegraph_llm \ + --cov-report=term \ + --cov-report=json:.workflow/quality-program/coverage/combined-baseline.json +``` + +## Ratchet Rules + +- Do not lower an existing local threshold for a touched ratchet area. +- For production bug fixes, add or update a regression test in the nearest unit, contract, integration, or smoke layer. +- For HugeGraph service-boundary changes, include a selected Layer B or Layer C command with `HUGEGRAPH_REQUIRED=true`. +- For new external-provider behavior, keep deterministic contract tests in default gates and reserve live-provider coverage for Layer D. diff --git a/docs/superpowers/plans/2026-05-31-hugegraph-ai-quality-program.md b/docs/superpowers/plans/2026-05-31-hugegraph-ai-quality-program.md index b8ae42e7..071fbc38 100644 --- a/docs/superpowers/plans/2026-05-31-hugegraph-ai-quality-program.md +++ b/docs/superpowers/plans/2026-05-31-hugegraph-ai-quality-program.md @@ -1359,7 +1359,7 @@ git commit -m "test(llm): add core pipeline smoke gates" -m "- add deterministic - Create/Update: `.workflow/quality-program/coverage/combined-baseline.json` - Create/Update: `.workflow/quality-program/checkpoints/07-coverage-ratchet.md` -- [ ] **Step G6.1: Create coverage ratchet documentation** +- [x] **Step G6.1: Create coverage ratchet documentation** Create `docs/quality/coverage-ratchet.md`: @@ -1383,7 +1383,7 @@ Create `docs/quality/coverage-ratchet.md`: - `hugegraph_llm.api.models` ``` -- [ ] **Step G6.2: Generate combined coverage baseline** +- [x] **Step G6.2: Generate combined coverage baseline** Run: @@ -1398,7 +1398,7 @@ uv run pytest hugegraph-python-client/src/tests hugegraph-llm/src/tests \ Expected: combined baseline file exists. Do not enforce a high threshold yet. -- [ ] **Step G6.3: Split client CI into layer jobs** +- [x] **Step G6.3: Split client CI into layer jobs** Modify `.github/workflows/hugegraph-python-client.yml` into at least: @@ -1416,7 +1416,7 @@ Required properties: - coverage artifact is uploaded when feasible ``` -- [ ] **Step G6.4: Split LLM CI into layer jobs** +- [x] **Step G6.4: Split LLM CI into layer jobs** Modify `.github/workflows/hugegraph-llm.yml` into at least: @@ -1435,7 +1435,7 @@ Required properties: - external tests are not default PR gates ``` -- [ ] **Step G6.5: Add local ratchet commands** +- [x] **Step G6.5: Add local ratchet commands** Document commands in `docs/quality/coverage-ratchet.md`: @@ -1446,7 +1446,7 @@ uv run pytest hugegraph-llm/src/tests/operators/llm_op -m "unit or contract" --c uv run pytest hugegraph-llm/src/tests/api -m "unit or contract" --cov=hugegraph_llm.api --cov-report=term ``` -- [ ] **Step G6.6: Run full local verification** +- [x] **Step G6.6: Run full local verification** Run: @@ -1462,7 +1462,7 @@ uv run pytest hugegraph-llm/src/tests/integration -m "smoke" -v --tb=short Expected: failures are either fixed or recorded with classification and next action. -- [ ] **Step G6.7: Write checkpoint and commit** +- [x] **Step G6.7: Write checkpoint and commit** Run:
