How to regression-test an LLM system

# .github/workflows/llm-regression.yml
name: llm-regression
on:
  pull_request:
    paths:
      - "prompts/**"
      - "src/llm/**"
      - "config/models.yaml"
      - "evals/**"
  schedule:
    - cron: "0 5 * * *"   # nightly: catches a provider-side model change with no PR

permissions:
  contents: read
  pull-requests: write

jobs:
  gate:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - run: pip install -r evals/requirements.txt

      - name: Run the suite, 3 samples per case
        run: python evals/run.py --suite regression --samples 3 --out evals/out/candidate.json
        env:
          MODEL_API_KEY: ${{ secrets.MODEL_API_KEY }}

      - name: Gate against the baseline from main
        run: pytest evals/test_regression_gate.py -q

      - name: Post the score diff on the PR
        if: always() && github.event_name == 'pull_request'
        run: |
          python evals/report.py evals/baseline.json evals/out/candidate.json > diff.md
          gh pr comment ${{ github.event.pull_request.number }} --body-file diff.md
        env:
          GH_TOKEN: ${{ github.token }}

# evals/test_regression_gate.py
import json
from pathlib import Path

import pytest

BASELINE = json.loads(Path("evals/baseline.json").read_text())
CANDIDATE = json.loads(Path("evals/out/candidate.json").read_text())
# Per evaluator: {"floor": 0.90, "tolerance": 0.03}. Set the tolerance from
# measured run-to-run noise on this suite, never by copying a number.
THRESHOLDS = json.loads(Path("evals/thresholds.json").read_text())


def pass_rate(run: dict, evaluator: str) -> float:
    # Every sample of every case is one verdict, so 3 samples give 3 votes.
    verdicts = [v for case in run["cases"] for v in case["verdicts"][evaluator]]
    return sum(verdicts) / len(verdicts)


@pytest.mark.parametrize("evaluator", sorted(THRESHOLDS))
def test_evaluator_holds(evaluator: str) -> None:
    t = THRESHOLDS[evaluator]
    base, cand = pass_rate(BASELINE, evaluator), pass_rate(CANDIDATE, evaluator)
    assert cand >= t["floor"], f"{evaluator}: {cand:.3f} is under its floor {t['floor']}"
    assert base - cand <= t["tolerance"], (
        f"{evaluator}: fell {base - cand:.3f} against main (tolerance {t['tolerance']})"
    )


def test_critical_cases_pass_every_sample() -> None:
    failed = [
        case["id"]
        for case in CANDIDATE["cases"]
        if case.get("critical") and not all(all(v) for v in case["verdicts"].values())
    ]
    assert not failed, f"critical cases failed: {failed}"