diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..386482e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +# Both images build from the repo root (see compose.yml). Keep secrets and +# local clutter out of them: `COPY . .` in the harness's Dockerfile would +# otherwise bake a local .env (Slack/Zebrunner/radiator tokens) into the image. +.env +.env.* +!.env.example +deploy/helm/*/values.*.yaml +.git +.github +.pytest_cache +.vscode +**/__pycache__ +*.egg-info +logs/ +test_results/ diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..3ec6ac1 --- /dev/null +++ b/.env.example @@ -0,0 +1,43 @@ +# Copy to .env (next to compose.yml) and fill in. .env is git-ignored; never +# commit real tokens. Docker Compose reads .env on its own. When running +# test-harness directly, set these in your shell instead. + +# --- Slack ------------------------------------------------------------------- +# Set all three to post the report to Slack; with any missing, the results are +# saved to test_results/ instead. Leave off --local. + +# Incoming webhook for the channel (the report message). +SLACK_WEBHOOK_URL=https://hooks.slack.com/services/your-webhook-path +# Bot token with the files:write scope (uploads the charts and result files). +# Invite the bot to the channel: /invite @your-app +SLACK_TOKEN=xoxb-your-bot-token +# The channel ID (from the channel's details, eg C0123ABCD), not its #name. +SLACK_CHANNEL=C0123ABCD + +# --- Optional: Zebrunner, alongside the new radiator -------------------------- +# Set both to report each run to Zebrunner as well. SLACK_CHANNEL then gets +# Zebrunner's report (with its link), and the new radiator's goes to +# RADIATOR_SLACK_* below, if set. +# ZE_BASE_URL=https://zebrunner.example.org +# ZE_REFRESH_TOKEN=your-zebrunner-refresh-token + +# --- Optional: a Slack channel of the new radiator's own ---------------------- +# While Zebrunner is configured (ZE_*), SLACK_CHANNEL is Zebrunner's and gets +# only the report it always did. Set these to post the new Information +# Radiator's reports (its link, pass-rate headline and charts), the run cycle +# boards and the weekly digest to a channel of their own. Without Zebrunner, +# SLACK_CHANNEL is the radiator's and these aren't needed. +# An incoming webhook posts to one channel only, so this needs its own; the +# bot token defaults to SLACK_TOKEN (invite the bot to this channel too). +# RADIATOR_SLACK_WEBHOOK_URL=https://hooks.slack.com/services/another-webhook-path +# RADIATOR_SLACK_CHANNEL=C0456EFGH +# RADIATOR_SLACK_TOKEN=xoxb-another-bot-token + +# --- Optional (Docker Compose) ----------------------------------------------- +# HARNESS_ARGS=load local_acceptance +# QUERY_PARAMETERS={"timeout": 300} +# RADIATOR_USERNAME=translator +# RADIATOR_PASSWORD=radiator +# RADIATOR_TOKEN=local-dev-token +# RADIATOR_PORT=8000 +# RADIATOR_EXCLUDED_ENVS=dev diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 229a8c2..69d0651 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -26,6 +26,65 @@ jobs: - name: Run tests and get output run: pytest + radiator-test: + name: Run the Information Radiator's tests + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: radiator_test + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: + - name: Checkout the repository + uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + name: Setup python + with: + python-version: '3.11' + cache: 'pip' + + - name: Install requirements + run: pip install -r radiator/requirements-test.txt + + - name: Check the migrations match the models + env: + RADIATOR_DATABASE_URL: postgresql+psycopg://postgres:postgres@localhost:5432/radiator_test + run: | + alembic -c radiator/alembic.ini upgrade head + alembic -c radiator/alembic.ini check + + - name: Run tests + env: + RADIATOR_TEST_DATABASE_URL: postgresql+psycopg://postgres:postgres@localhost:5432/radiator_test + run: pytest radiator/tests + + helm-lint: + name: Lint and render the Helm charts + runs-on: ubuntu-latest + steps: + - name: Checkout the repository + uses: actions/checkout@v4 + + - uses: azure/setup-helm@v4 + with: + version: v3.16.4 + + - name: Lint and render each chart with its example values + run: | + for chart in deploy/helm/*/; do + helm lint "$chart" -f "$chart/values.example.yaml" + helm template ci "$chart" -f "$chart/values.example.yaml" > /dev/null + done + check-format: name: Check that code matches Black formatter runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 15e87a0..e754514 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,3 +23,13 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} registry: ghcr.io tags: latest,${{ steps.get_version.outputs.VERSION }} + - name: Push the Information Radiator to GitHub Packages + uses: docker/build-push-action@v1 + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: ghcr.io + repository: translatorsri/testharness-radiator + dockerfile: radiator/Dockerfile + path: . + tags: latest,${{ steps.get_version.outputs.VERSION }} diff --git a/.gitignore b/.gitignore index 6a03c19..3f50ed1 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,7 @@ scripts/ .vscode test_results/ test_report.json +# Helm values for a real deployment: copies of the charts' values.example.yaml +deploy/helm/*/values.*.yaml +!deploy/helm/*/values.example.yaml +.pytest_cache/ diff --git a/README.md b/README.md index d4c7e47..8f707ad 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,139 @@ saved to `--output_dir` automatically. Likewise, if the Information Radiator isn't configured (no `ZE_BASE_URL` / `ZE_REFRESH_TOKEN`), the harness falls back to a local reporter. +### The new Information Radiator +A replacement for the Zebrunner-based Information Radiator lives in this repo, +in `radiator/` (full details in +[`docs/information-radiator.md`](docs/information-radiator.md)). It gets +structured results instead of labels and log lines: for each test asset, every +agent's status, whether the expected answer was found and its rank/score, the +number of results, the HTTP status, and (for direct queries) the response +time. Its dashboard shows each run, what changed since the previous one, each +asset's history, agent pass-rate trends, and performance trends. + +#### Trying it out with Docker Compose +With Docker installed, from the repo root: +- `docker compose up --build` + +That starts Postgres, sets up its tables, starts the radiator, and then runs a +small test suite (`local_acceptance`) that reports to it. When the suite +finishes, the harness container exits; the radiator keeps running at +. Log in with `translator` / `radiator`. + +The suite queries the real Translator services (the ARS on ci), so the run +needs internet access and takes a few minutes. It reports to the new radiator +only, unless you put `ZE_BASE_URL` and `ZE_REFRESH_TOKEN` in `.env` to report +to Zebrunner as well (see +[A Slack channel for the new radiator](#a-slack-channel-for-the-new-radiator) +for where Slack posts go then). + +Everything is configurable from a `.env` file next to `compose.yml`; copy +`.env.example` to `.env` to start from the Slack settings: + +| Setting | Default | | +| --- | --- | --- | +| `HARNESS_ARGS` | `load local_acceptance` | What the harness runs, eg `download sprint_4_tests`, or `--query_type MVP1 download sprint_4_tests` | +| `RADIATOR_USERNAME` / `RADIATOR_PASSWORD` | `translator` / `radiator` | The login | +| `RADIATOR_TOKEN` | `local-dev-token` | The API token the harness uploads with | +| `RADIATOR_PORT` | `8000` | Where the radiator is published on your machine | +| `QUERY_PARAMETERS` | unset | Parameters sent with every query, eg `{"timeout": 300}` (see [Sending parameters with every query](#sending-parameters-with-every-query)) | +| `SLACK_WEBHOOK_URL`, `SLACK_TOKEN`, `SLACK_CHANNEL` | unset | Set all three to also post the report, with the radiator's charts, to Slack | +| `ZE_BASE_URL`, `ZE_REFRESH_TOKEN` | unset | Also report to Zebrunner; `SLACK_CHANNEL` then gets only Zebrunner's report, and the radiator's go to `RADIATOR_SLACK_*` (or nowhere, if that isn't set) | +| `RADIATOR_SLACK_WEBHOOK_URL`, `RADIATOR_SLACK_CHANNEL` (`RADIATOR_SLACK_TOKEN`) | unset | A channel of the radiator's own (see [A Slack channel for the new radiator](#a-slack-channel-for-the-new-radiator)) | + +Then: +- **Run another suite** against the same radiator, without restarting it: + `docker compose run --rm harness test-harness download sprint_4_tests` +- **Run a cycle** (acceptance, then pathfinder, then performance) and get its + board: `docker compose run --rm harness test-harness-cycle --download + --acceptance sprint_4_tests --pathfinder pathfinder_tests` (see + [Run cycles](#run-cycles-and-the-weekly-digest)). +- **See the dashboard with made-up history** instead (eg to look around without + waiting for real runs): `docker compose run --rm radiator python -m radiator.demo`. + Don't do this in a radiator you'll keep: the demo runs mix in with real ones. +- **Stop it**: `docker compose down` keeps the results (in the `radiator-db` + volume) for next time; `docker compose down -v` wipes them. + +If the harness exits with `Permission denied: '/app/logs/harness.log'` (a Linux +host where `./logs` isn't writable by the container's user), run +`chmod a+w logs` (and remove `logs/harness.log` if it exists and isn't +yours), then `docker compose up` again. + +#### Reporting to a radiator from the harness +Point the harness at a radiator with `RADIATOR_URL` and `RADIATOR_TOKEN` (or +`--radiator_url` / `--radiator_token`). If people open the radiator at a +different address than the harness uploads to (eg an in-cluster service name), +set `RADIATOR_PUBLIC_URL` for the links in the logs and Slack. Until the new +radiator takes over, the harness reports to it alongside Zebrunner, if that's +configured too. + +Results are posted in batches; an unreachable radiator never fails a run. +Whenever a run isn't uploaded (`--local`, the radiator isn't configured, or an +upload failed) it's saved to `--output_dir` as `radiator_.json` +instead, which can be uploaded later: +- `test-harness-radiator push test_results/radiator_.json` + +Uploads are idempotent, so pushing a run that partly made it is safe. The +radiator never depends on Zebrunner or Slack: if either fails, the run still +reaches the radiator (or is saved for it). + +When the radiator has the run, the Slack report also gets a headline against +the previous run and a chart of the last 30 runs: for acceptance runs, the pass +rate, regressions, and fixes, charted per agent; for performance runs, each +service's max sustainable concurrency and checkpoint verdict. Runs of a +[run cycle](#run-cycles-and-the-weekly-digest) also get the cycle's board when +the cycle finishes, and the radiator's **Weekly** page and Monday digest cover +every environment. + +The dashboard's **Run grid** page shows a suite's runs in every environment +over time, colored by whether the pass rate went up, down, or held. The `dev` +environment is shown greyed out as excluded there; set `RADIATOR_EXCLUDED_ENVS` +on the radiator (comma-separated) to change which. + +#### A Slack channel for the new radiator +Which channel gets what depends on whether Zebrunner is still configured +(`ZE_BASE_URL` and `ZE_REFRESH_TOKEN`): + +- **Zebrunner configured:** `SLACK_CHANNEL` is still Zebrunner's. It gets the + report it got before the new radiator (the Zebrunner link, the result summary + and the result files) and nothing of the new radiator's. The new radiator's + report goes to a channel of its own, if you give it one (below). +- **No Zebrunner** (after the cutover, or in Docker Compose without `ZE_*` in + `.env`): `SLACK_CHANNEL` is the new radiator's and gets its report. + +This keys off whether Zebrunner is *configured*, not whether it answered, so a +Zebrunner outage never switches the old channel over to the new dashboard. + +To give the new radiator its own channel while both run, set: + +| Setting | | +|---|---| +| `RADIATOR_SLACK_WEBHOOK_URL` | An incoming webhook for the new channel (a webhook posts to one channel only) | +| `RADIATOR_SLACK_CHANNEL` | The new channel's ID, eg `C0456EFGH` | +| `RADIATOR_SLACK_TOKEN` | Optional: defaults to `SLACK_TOKEN`. Invite that bot to the new channel too | + +That channel then gets the new radiator's report for each run (its link, the +headline against the previous run, the history charts and the result files), +the [run cycle boards and the weekly digest](#run-cycles-and-the-weekly-digest). +Each run still runs once and reports to both dashboards. A failure posting to +the new channel (eg the bot isn't in it) is logged and never costs the old +channel its report. Without it, the new radiator still records every run; it +just posts nothing to Slack while Zebrunner is configured. + +When you retire Zebrunner, drop the `ZE_*` settings, point `SLACK_*` at the +new channel, and drop `RADIATOR_SLACK_*`. + +#### Deploying it +To run it for real, in the same Kubernetes namespace as the harness, use the +Helm charts in [`deploy/helm`](deploy/helm/README.md): one for the radiator +(and, if you need it, a small Postgres), one for the harness's scheduled run +cycles, digest and sweeps. Copy each chart's `values.example.yaml` to a +git-ignored `values..yaml` and install. The plain manifests in `deploy/` +show the same setup without Helm; see +[Deploying](docs/information-radiator.md#deploying). To bring over the +history from Zebrunner, see +[Importing from Zebrunner](docs/information-radiator.md#importing-from-zebrunner). + ### Running local test files `test-harness download ` fetches the suites from [NCATSTranslator/Tests](https://github.com/NCATSTranslator/Tests) every run, @@ -88,6 +221,64 @@ whenever `--query_type` is given. Saved result filenames are prefixed with the query type (eg `MVP1_test_report.json`) so runs of different slices stay distinguishable. +### TRAPI version +The harness speaks **TRAPI 2.0 and 1.6**. Queries are built with +[`translator_tom`](https://github.com/NCATSTranslator/TRAPIObjectModeling) +(TOM), the Translator-wide TRAPI object model, so each query is validated +against the spec before it is sent. `--trapi_version` picks the version the +queries are written in and which services are taken from the SmartAPI +registry (only those registered with the same `x-trapi` major.minor): +- `auto` (the default): the version the environment's ARS is registered with + in SmartAPI (or, with `--target`, that service's), the newest the harness + speaks if it's registered with more than one. While environments move to + 2.0 one at a time, each run follows its environment with no configuration. + The log says what it chose and why; if nothing is registered, or SmartAPI + can't be read, it falls back to 2.0 with a warning. +- `--trapi_version 1.6.0` or `--trapi_version 2.0.0` to choose one, eg if an + environment's SmartAPI registration and its deployment disagree for a while. + In a run cycle, pass it after the `--` (or, with the Helm chart, in that + environment's `harnessArgs`). + +Each run's TRAPI version is recorded in the Information Radiator, shown on the +run and in the runs list, and the start of the Slack report says it. When a +run's version differs from the previous run's (an environment that just moved +to 2.0), the run page, What changed, the cycle board's flags and the Slack +headline all say so: what changed may be the upgrade, not the services. + +What 2.0 changed in the queries the harness sends: an MVP2 query's qualifiers +go in the query edge's `constraints.qualifiers` as one +`{qualifier_type_id: qualifier_value}` mapping (1.6's `qualifier_constraints` +is rejected by 2.0 services), and a Pathfinder query is an ordinary query graph +with `paths` and no `edges`. Responses are read in either shape — a 2.0 +binding is one object with an `ids` list, a 1.x binding a list of objects each +with an `id` — since the ARS can return either. + +### Sending parameters with every query +To send the same parameters with every query of a run, such as a longer +timeout or a request to skip caches, pass them as a JSON object: +- `test-harness --query_parameters '{"timeout": 300, "bypass_cache": true}' download ` + +or keep them in a file and pass `--query_parameters @params.json`. The +`QUERY_PARAMETERS` environment variable does the same, which is handier in +`.env` for Docker Compose or in a CronJob; the flag wins if both are set. + +Where they go depends on the TRAPI version: +- **TRAPI 2.0**: in the query's `parameters` object. The + standard ones are `timeout` (seconds), `log_level` (`ERROR`, `WARNING`, + `INFO` or `DEBUG`) and `bypass_cache`; a service can define its own as well, + and those are passed through as given. +- **TRAPI 1.6**: as top-level fields of the query. + 1.6 only has `log_level` and `bypass_cache`, so anything else is refused. + +They're checked before the run starts: a value of the wrong type, an unknown +log level, or a parameter the TRAPI version can't carry stops the run with a +message saying which, instead of failing every query. The run's parameters are +shown in its Slack message and recorded in the Information Radiator, on the run +page. + +They go to whatever the harness queries (the ARS, or the `--target_url` +service). Performance tests don't use them: HelmsDeep sends its own queries. + ### Result files The acceptance results CSV has a row per test asset, with the asset's name, its Information Radiator URL, the ARS pk, and the test case/asset ids. Every agent @@ -241,6 +432,50 @@ four-service sweep is roughly 70 min for the ARS plus 61 min per ARA (~4.2 hours); the single-class default profile is ~58 + ~35 × 3 (~2.7 hours). Nothing overlaps, by design. +### Run cycles and the weekly digest +A run cycle is a weekly look at one environment: acceptance, then pathfinder, +then a HelmsDeep performance run against that environment's ARS, one after +another so they never load the services at the same time (about 2 hours with +the `mixed` profile). The intended schedule is one environment a day: dev on +Monday, ci on Tuesday, test on Wednesday, prod on Thursday. + +``` +test-harness-cycle --download \ + --acceptance sprint_4_tests --pathfinder pathfinder_tests \ + --performance performance_tests \ + --performance_target ars=https://ars.ci.transltr.io \ + --performance_profile mixed \ + -- --log_level INFO +``` + +Each step is an ordinary harness run with its usual Slack report. Leave out any +step you don't want; anything after `--` goes to every step (eg +`--query_parameters`). When the last step finishes, the cycle posts the +Information Radiator's **board**: each test against its previous run in that +environment (pass rate or max concurrency, the change, regressions and fixes), +with a verdict of *Regressed*, *Improved*, *Mixed* or *Steady*, the regressions +and fixes by agent, and **flags** for oddities a pass rate hides: + +- an agent returned no results on 3+ assets that had results last run; +- an agent errored on 3+ assets that didn't error last run; +- an agent's median response time at least doubled, and by 5s or more; +- the expected answer fell 30+ places while still passing; +- the ARS's max sustainable concurrency fell by 25% or more, its run failed, or + it missed checkpoints it passed last run. + +The cycle needs `RADIATOR_URL`/`RADIATOR_TOKEN` for the board (without them, +the steps still run and report). Its exit status works like the sweep's. +The [test-harness Helm chart](deploy/helm/README.md) schedules a cycle per +environment; [`deploy/cycle.example.yaml`](deploy/cycle.example.yaml) is the +same CronJob without Helm. + +Every Monday morning, `test-harness-radiator digest` posts the **weekly digest**: +each environment's latest runs of acceptance, pathfinder and performance over +the past week, against the run before, with their flags. It reads from the +radiator only, so it's quick; the Helm chart schedules it too (or +[`deploy/digest.example.yaml`](deploy/digest.example.yaml) without Helm). The same view is the radiator's **Weekly** page, with a page per +cycle. + ### Overriding the target service Tests specify which component to run against (`ars`, `ara`, ...), and the harness normally resolves those components to deployed services through the diff --git a/compose.yml b/compose.yml index 3b4fa66..9797335 100644 --- a/compose.yml +++ b/compose.yml @@ -1,6 +1,71 @@ -version: '3.7' +# Try the harness and the new Information Radiator together, locally: +# +# docker compose up --build +# +# starts Postgres, migrates it, starts the radiator on http://localhost:8000, +# then runs a test suite that reports to it. When the run finishes the +# harness exits; the radiator keeps running. Log in with RADIATOR_USERNAME / +# RADIATOR_PASSWORD (translator / radiator unless set in .env). +# +# This reports to the new radiator, and also to Zebrunner if ZE_BASE_URL and +# ZE_REFRESH_TOKEN are in .env. Slack is off unless the SLACK_* settings are in +# .env. The defaults below are for local use only. + +x-radiator-env: &radiator-env + RADIATOR_DATABASE_URL: postgresql+psycopg://radiator:radiator@radiator-db:5432/radiator + RADIATOR_API_TOKEN: ${RADIATOR_TOKEN:-local-dev-token} + RADIATOR_USERNAME: ${RADIATOR_USERNAME:-translator} + RADIATOR_PASSWORD: ${RADIATOR_PASSWORD:-radiator} + RADIATOR_SESSION_SECRET: ${RADIATOR_SESSION_SECRET:-local-dev-session-secret} + # plain http on localhost + RADIATOR_SECURE_COOKIES: "false" + RADIATOR_EXCLUDED_ENVS: ${RADIATOR_EXCLUDED_ENVS:-dev} services: + radiator-db: + image: postgres:16 + environment: + POSTGRES_USER: radiator + POSTGRES_PASSWORD: radiator + POSTGRES_DB: radiator + volumes: + - radiator-db:/var/lib/postgresql/data + healthcheck: + test: ["CMD", "pg_isready", "-U", "radiator"] + interval: 2s + timeout: 5s + retries: 30 + + # brings the schema up to date, then exits + radiator-migrate: + image: testharness-radiator + build: + context: . + dockerfile: radiator/Dockerfile + command: alembic -c radiator/alembic.ini upgrade head + environment: *radiator-env + depends_on: + radiator-db: + condition: service_healthy + + radiator: + image: testharness-radiator + environment: *radiator-env + ports: + - "${RADIATOR_PORT:-8000}:8000" + depends_on: + radiator-migrate: + condition: service_completed_successfully + healthcheck: + test: + - CMD + - python + - -c + - "import urllib.request; urllib.request.urlopen('http://localhost:8000/healthz')" + interval: 3s + timeout: 5s + retries: 20 + harness: image: test-harness container_name: test-harness @@ -8,10 +73,31 @@ services: context: . volumes: - ./logs:/app/logs - command: test-harness download sprint_4_tests + # A small local suite by default; set HARNESS_ARGS in .env to run another, + # eg HARNESS_ARGS="download sprint_4_tests". It still queries the real + # Translator services, so expect a run to take a few minutes. + command: test-harness ${HARNESS_ARGS:-load local_acceptance} environment: - SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL} - SLACK_TOKEN: ${SLACK_TOKEN} - SLACK_CHANNEL: ${SLACK_CHANNEL} - ZE_BASE_URL: ${ZE_BASE_URL} - ZE_REFRESH_TOKEN: ${ZE_REFRESH_TOKEN} + RADIATOR_URL: http://radiator:8000 + RADIATOR_PUBLIC_URL: http://localhost:${RADIATOR_PORT:-8000} + RADIATOR_TOKEN: ${RADIATOR_TOKEN:-local-dev-token} + # parameters sent with every query, eg {"timeout": 300} + QUERY_PARAMETERS: ${QUERY_PARAMETERS:-} + SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL:-} + SLACK_TOKEN: ${SLACK_TOKEN:-} + SLACK_CHANNEL: ${SLACK_CHANNEL:-} + # a channel of the new radiator's own (see .env.example) + RADIATOR_SLACK_WEBHOOK_URL: ${RADIATOR_SLACK_WEBHOOK_URL:-} + RADIATOR_SLACK_CHANNEL: ${RADIATOR_SLACK_CHANNEL:-} + RADIATOR_SLACK_TOKEN: ${RADIATOR_SLACK_TOKEN:-} + # Zebrunner, alongside the new radiator: unset, the run reports to the + # new radiator only, and SLACK_CHANNEL gets its report + ZE_BASE_URL: ${ZE_BASE_URL:-} + ZE_REFRESH_TOKEN: ${ZE_REFRESH_TOKEN:-} + PYTHONUNBUFFERED: "1" + depends_on: + radiator: + condition: service_healthy + +volumes: + radiator-db: diff --git a/deploy/cronjob.example.yaml b/deploy/cronjob.example.yaml index f3ad6e6..ffee114 100644 --- a/deploy/cronjob.example.yaml +++ b/deploy/cronjob.example.yaml @@ -1,3 +1,6 @@ +# The Helm charts in deploy/helm/ are the maintained way to deploy; these plain +# manifests are kept as a reference for clusters without Helm. +# # Example CronJob for a sequential performance sweep. # # One Job, four harness runs, strictly in order: the ARS first, then each ARA. @@ -109,12 +112,35 @@ spec: - name: SLACK_CHANNEL valueFrom: secretKeyRef: {name: test-harness-secrets, key: slack-channel} + # Optional: a channel of the new radiator's own, alongside + # Zebrunner (see the README's "A Slack channel for the new + # radiator"). The token defaults to slack-token. + - name: RADIATOR_SLACK_WEBHOOK_URL + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: radiator-slack-webhook-url, optional: true} + - name: RADIATOR_SLACK_CHANNEL + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: radiator-slack-channel, optional: true} - name: ZE_BASE_URL valueFrom: secretKeyRef: {name: test-harness-secrets, key: ze-base-url} - name: ZE_REFRESH_TOKEN valueFrom: secretKeyRef: {name: test-harness-secrets, key: ze-refresh-token} + # The new Information Radiator, reported to alongside + # Zebrunner until it takes over. Optional: without it each run + # is saved to --output_dir in the pod, which is lost with the + # pod unless that directory is a mounted volume. + - name: RADIATOR_URL + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: radiator-url, optional: true} + - name: RADIATOR_TOKEN + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: radiator-token, optional: true} + # Uploads can go to the in-cluster service; links people + # click (in Slack) need the radiator's public address. + - name: RADIATOR_PUBLIC_URL + value: https://radiator.example.org resources: # The load generator is the workload here: locust drives up to diff --git a/deploy/cycle.example.yaml b/deploy/cycle.example.yaml new file mode 100644 index 0000000..0c69214 --- /dev/null +++ b/deploy/cycle.example.yaml @@ -0,0 +1,114 @@ +# The Helm charts in deploy/helm/ are the maintained way to deploy; these plain +# manifests are kept as a reference for clusters without Helm. +# +# Example CronJobs for weekly run cycles: one environment a day, Monday to +# Thursday (dev, ci, test, prod), so a regression shows up in dev before it +# reaches prod and Friday is free for re-runs. +# +# Each cycle runs acceptance (~50 min), then pathfinder, then a HelmsDeep +# performance run against that environment's ARS (~70 min with the mixed +# profile), one after another so they never load the services at the same +# time. Each step posts its own Slack report as usual; when the cycle finishes +# it posts the Information Radiator's board for the whole cycle. +# +# This file is the dev CronJob in full. For the others, copy it and change the +# four lines marked CHANGE PER ENVIRONMENT: +# +# name schedule (America/New_York) ARS +# test-harness-cycle-dev "52 1 * * 1" Monday https://ars.dev.transltr.io +# test-harness-cycle-ci "52 1 * * 2" Tuesday https://ars.ci.transltr.io +# test-harness-cycle-test "52 1 * * 3" Wednesday https://ars.test.transltr.io +# test-harness-cycle-prod "52 1 * * 4" Thursday https://ars.transltr.io +# +# and the suites: the environment comes from the test cases, so each +# environment runs the suites written for it. +# +# The image tag, namespace, secret names and suite names are placeholders. +apiVersion: batch/v1 +kind: CronJob +metadata: + name: test-harness-cycle-dev # CHANGE PER ENVIRONMENT +spec: + schedule: "52 1 * * 1" # CHANGE PER ENVIRONMENT + timeZone: "America/New_York" # requires Kubernetes >= 1.27 + # Never two cycles at once: they would load the same services together. + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + # Never retry: a retry re-runs every step and re-posts every report. + # The Job only fails when a step couldn't run at all, never on a test + # result. + backoffLimit: 0 + # A cycle takes ~2 h; 4 h leaves room without letting a wedged run hold + # the next day's environment. + activeDeadlineSeconds: 14400 + template: + spec: + restartPolicy: Never + terminationGracePeriodSeconds: 120 + containers: + - name: test-harness + image: ghcr.io/translatorsri/testharness:latest + imagePullPolicy: IfNotPresent + command: ["test-harness-cycle"] + args: + - "--download" + - "--acceptance" + - "sprint_4_tests" # CHANGE PER ENVIRONMENT: its acceptance suite + - "--pathfinder" + - "pathfinder_tests" # its pathfinder suite + - "--performance" + - "performance_tests" # its performance suite + - "--performance_target" + - "ars=https://ars.dev.transltr.io" # CHANGE PER ENVIRONMENT + # The only profile with pass/fail checkpoints. + - "--performance_profile" + - "mixed" + # Everything after `--` goes to every step. + - "--" + - "--log_level" + - "INFO" + env: + - name: PYTHONUNBUFFERED + value: "1" + - name: SLACK_WEBHOOK_URL + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: slack-webhook-url} + - name: SLACK_TOKEN + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: slack-token} + - name: SLACK_CHANNEL + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: slack-channel} + # Optional: a channel of the new radiator's own, alongside + # Zebrunner (see the README's "A Slack channel for the new + # radiator"). The token defaults to slack-token. + - name: RADIATOR_SLACK_WEBHOOK_URL + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: radiator-slack-webhook-url, optional: true} + - name: RADIATOR_SLACK_CHANNEL + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: radiator-slack-channel, optional: true} + # The board comes from the radiator: without it, each step + # still reports, but there's no board at the end. + - name: RADIATOR_URL + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: radiator-url} + - name: RADIATOR_TOKEN + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: radiator-token} + - name: RADIATOR_PUBLIC_URL + value: https://radiator.example.org + resources: + # HelmsDeep's load generator runs in this container during the + # performance step; give it real CPU (see cronjob.example.yaml). + requests: {cpu: "2", memory: "2Gi"} + limits: {cpu: "2", memory: "4Gi"} + volumeMounts: + - name: results + mountPath: /app/test_results + volumes: + - name: results + emptyDir: {} diff --git a/deploy/digest.example.yaml b/deploy/digest.example.yaml new file mode 100644 index 0000000..e65762e --- /dev/null +++ b/deploy/digest.example.yaml @@ -0,0 +1,59 @@ +# The Helm charts in deploy/helm/ are the maintained way to deploy; these plain +# manifests are kept as a reference for clusters without Helm. +# +# Example CronJob for the weekly digest: every Monday morning, one Slack post +# with each environment's latest runs of acceptance, pathfinder and +# performance over the past week, each against the run before, and the flags. +# +# It only reads from the Information Radiator, so it's quick and light. +# The image tag and secret names are placeholders. +apiVersion: batch/v1 +kind: CronJob +metadata: + name: test-harness-weekly-digest +spec: + schedule: "52 8 * * 1" # Monday, 08:52 + timeZone: "America/New_York" # requires Kubernetes >= 1.27 + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 1 + activeDeadlineSeconds: 600 + template: + spec: + restartPolicy: Never + containers: + - name: digest + image: ghcr.io/translatorsri/testharness:latest + imagePullPolicy: IfNotPresent + command: ["test-harness-radiator", "digest"] + env: + - name: SLACK_WEBHOOK_URL + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: slack-webhook-url} + - name: SLACK_TOKEN + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: slack-token} + - name: SLACK_CHANNEL + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: slack-channel} + # Optional: a channel of the new radiator's own, alongside + # Zebrunner (see the README's "A Slack channel for the new + # radiator"). The token defaults to slack-token. + - name: RADIATOR_SLACK_WEBHOOK_URL + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: radiator-slack-webhook-url, optional: true} + - name: RADIATOR_SLACK_CHANNEL + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: radiator-slack-channel, optional: true} + - name: RADIATOR_URL + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: radiator-url} + - name: RADIATOR_TOKEN + valueFrom: + secretKeyRef: {name: test-harness-secrets, key: radiator-token} + - name: RADIATOR_PUBLIC_URL + value: https://radiator.example.org + resources: + requests: {cpu: "100m", memory: "128Mi"} + limits: {memory: "256Mi"} diff --git a/deploy/helm/README.md b/deploy/helm/README.md new file mode 100644 index 0000000..b479540 --- /dev/null +++ b/deploy/helm/README.md @@ -0,0 +1,86 @@ +# Helm charts + +Two charts, installed into the same namespace: + +| Chart | What it deploys | +|---|---| +| [`radiator`](radiator) | The Information Radiator: the web app (with a migration init container), its Service, an optional Ingress, an optional small Postgres, and an optional one-off Zebrunner import Job | +| [`test-harness`](test-harness) | The scheduled runs: a weekly run cycle per environment, the Monday digest, and an optional performance sweep (eg of the ARAs) | + +## Values + +Each chart's `values.yaml` holds the defaults; don't edit it for a deployment. +Copy `values.example.yaml` to `values..yaml` in the same directory and +fill that in. Those copies are git-ignored (only `values.example.yaml` is +tracked), so `values.prod.yaml` or `values.ci.yaml` never end up in a commit. + +``` +cp deploy/helm/radiator/values.example.yaml deploy/helm/radiator/values.prod.yaml +cp deploy/helm/test-harness/values.example.yaml deploy/helm/test-harness/values.prod.yaml +``` + +Secrets can go in two ways. The examples use the first: + +- **Create the Secrets yourself** and name them in `existingSecret` (and + `postgres.existingSecret`). The `kubectl create secret` commands are in each + example file. Nothing secret is then in any values file. +- **Let the chart create them**: leave `existingSecret` empty and fill in + `secrets`. The copy is git-ignored, but it's still a file with passwords in + it, so keep it somewhere safe. + +## Installing + +The radiator first, since the harness reports to it: + +``` +kubectl create namespace translator # if needed + +helm upgrade --install radiator deploy/helm/radiator \ + -n translator -f deploy/helm/radiator/values.prod.yaml + +helm upgrade --install test-harness deploy/helm/test-harness \ + -n translator -f deploy/helm/test-harness/values.prod.yaml +``` + +With a release named `radiator`, the radiator's Service is `radiator`, which is +what the harness example's `radiator.url` (`http://radiator`) expects; with +another release name, use the in-cluster URL the radiator's install notes print. +The harness's `radiator-token` is the radiator's `api-token`. + +Upgrades are the same command. The radiator migrates its database before it +starts, every time; that's a no-op when the schema is current. + +## The schedule + +`cycles.environments` in the harness values is one CronJob per environment. The +example runs dev on Monday, ci on Tuesday, test on Wednesday and prod on +Thursday, each at 01:52 in `timeZone`, and the digest on Monday at 08:52. The +environment comes from the test cases, so each environment names the suites +written for it. Any setting under `cycles.defaults` can be overridden per +environment; `performance: ""` leaves the performance step out (eg for prod). + +To run one now, without waiting for its day: + +``` +kubectl -n translator create job --from=cronjob/test-harness-cycle-ci cycle-ci-now +kubectl -n translator create job --from=cronjob/test-harness-weekly-digest digest-now +``` + +`suspend: true` on an environment (or `digest.suspend`, `sweep.suspend`) keeps +its CronJob but stops it from running. + +## Importing Zebrunner's history + +Once the radiator is up, set `zebrunnerImport.enabled: true` (try +`args: ["--dry-run", "--limit", "5"]` first) and upgrade; the Job runs once per +upgrade while it's enabled. Turn it off again afterwards. See +[Importing from Zebrunner](../../docs/information-radiator.md#importing-from-zebrunner). + +## Checking a change + +``` +helm lint deploy/helm/radiator -f deploy/helm/radiator/values.example.yaml +helm template radiator deploy/helm/radiator -f deploy/helm/radiator/values.example.yaml +``` + +CI lints and renders both charts with their example values. diff --git a/deploy/helm/radiator/.helmignore b/deploy/helm/radiator/.helmignore new file mode 100644 index 0000000..bca6ed2 --- /dev/null +++ b/deploy/helm/radiator/.helmignore @@ -0,0 +1,2 @@ +# copies of values.example.yaml stay out of the chart package +values.*.yaml diff --git a/deploy/helm/radiator/Chart.yaml b/deploy/helm/radiator/Chart.yaml new file mode 100644 index 0000000..f50b02e --- /dev/null +++ b/deploy/helm/radiator/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: radiator +description: The Translator Information Radiator, the dashboard the Test Harness reports to +type: application +version: 0.1.0 +# The image tag when values don't set one. +appVersion: "latest" diff --git a/deploy/helm/radiator/templates/NOTES.txt b/deploy/helm/radiator/templates/NOTES.txt new file mode 100644 index 0000000..d353e4d --- /dev/null +++ b/deploy/helm/radiator/templates/NOTES.txt @@ -0,0 +1,13 @@ +The Information Radiator is deployed as {{ include "radiator.fullname" . }}. +{{- if .Values.ingress.enabled }} + +Open it at http{{ if .Values.ingress.tlsSecretName }}s{{ end }}://{{ .Values.ingress.host }} +{{- else }} + +No ingress is enabled. To look at it: + kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "radiator.fullname" . }} 8000:{{ .Values.service.port }} +{{- end }} + +Point the harness at it (the test-harness chart's radiator.url): + http://{{ include "radiator.fullname" . }}.{{ .Release.Namespace }}.svc{{ if ne (int .Values.service.port) 80 }}:{{ .Values.service.port }}{{ end }} +and give it the api-token from {{ include "radiator.secretName" . }} as RADIATOR_TOKEN. diff --git a/deploy/helm/radiator/templates/_helpers.tpl b/deploy/helm/radiator/templates/_helpers.tpl new file mode 100644 index 0000000..c94c8ef --- /dev/null +++ b/deploy/helm/radiator/templates/_helpers.tpl @@ -0,0 +1,42 @@ +{{- define "radiator.fullname" -}} +{{- if contains .Chart.Name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{- define "radiator.labels" -}} +app.kubernetes.io/name: {{ .Chart.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version }} +{{- end -}} + +{{- define "radiator.selector" -}} +app.kubernetes.io/name: {{ .Chart.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: web +{{- end -}} + +{{- define "radiator.image" -}} +{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }} +{{- end -}} + +{{- define "radiator.secretName" -}} +{{ .Values.existingSecret | default (printf "%s-secrets" (include "radiator.fullname" .)) }} +{{- end -}} + +{{- define "radiator.postgresName" -}} +{{ include "radiator.fullname" . }}-postgres +{{- end -}} + +{{- define "radiator.postgresSecretName" -}} +{{ .Values.postgres.existingSecret | default (include "radiator.postgresName" .) }} +{{- end -}} + +{{- define "radiator.secretEnv" -}} +- name: RADIATOR_DATABASE_URL + valueFrom: {secretKeyRef: {name: {{ include "radiator.secretName" . }}, key: database-url}} +{{- end -}} diff --git a/deploy/helm/radiator/templates/deployment.yaml b/deploy/helm/radiator/templates/deployment.yaml new file mode 100644 index 0000000..0ca1434 --- /dev/null +++ b/deploy/helm/radiator/templates/deployment.yaml @@ -0,0 +1,76 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "radiator.fullname" . }} + labels: {{- include "radiator.labels" . | nindent 4 }} +spec: + # One replica: the login throttle is in memory, and the load is a handful + # of people looking at a dashboard. Recreate so a migration never runs + # against a database an old pod is still using. + replicas: 1 + strategy: {type: Recreate} + selector: + matchLabels: {{- include "radiator.selector" . | nindent 6 }} + template: + metadata: + labels: + {{- include "radiator.selector" . | nindent 8 }} + annotations: + # restart when the chart-managed secret changes + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.podSecurityContext }} + securityContext: {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + # Bring the schema up to date before the app starts. + - name: migrate + image: {{ include "radiator.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["alembic", "-c", "radiator/alembic.ini", "upgrade", "head"] + env: + {{- include "radiator.secretEnv" . | nindent 12 }} + resources: {{- toYaml .Values.migrate.resources | nindent 12 }} + containers: + - name: radiator + image: {{ include "radiator.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - {name: http, containerPort: 8000} + env: + {{- include "radiator.secretEnv" . | nindent 12 }} + - name: RADIATOR_API_TOKEN + valueFrom: {secretKeyRef: {name: {{ include "radiator.secretName" . }}, key: api-token}} + - name: RADIATOR_USERNAME + valueFrom: {secretKeyRef: {name: {{ include "radiator.secretName" . }}, key: username}} + - name: RADIATOR_PASSWORD + valueFrom: {secretKeyRef: {name: {{ include "radiator.secretName" . }}, key: password}} + - name: RADIATOR_SESSION_SECRET + valueFrom: {secretKeyRef: {name: {{ include "radiator.secretName" . }}, key: session-secret}} + - name: RADIATOR_SECURE_COOKIES + value: {{ .Values.config.secureCookies | quote }} + - name: RADIATOR_EXCLUDED_ENVS + value: {{ .Values.config.excludedEnvs | quote }} + - name: RADIATOR_LOG_LEVEL + value: {{ .Values.config.logLevel | quote }} + readinessProbe: + httpGet: {path: /healthz, port: http} + periodSeconds: 10 + livenessProbe: + # generous: a database blip shouldn't restart the app + httpGet: {path: /healthz, port: http} + periodSeconds: 30 + failureThreshold: 6 + resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deploy/helm/radiator/templates/ingress.yaml b/deploy/helm/radiator/templates/ingress.yaml new file mode 100644 index 0000000..86135ba --- /dev/null +++ b/deploy/helm/radiator/templates/ingress.yaml @@ -0,0 +1,29 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "radiator.fullname" . }} + labels: {{- include "radiator.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- if .Values.ingress.tlsSecretName }} + tls: + - hosts: [{{ .Values.ingress.host | quote }}] + secretName: {{ .Values.ingress.tlsSecretName }} + {{- end }} + rules: + - host: {{ required "ingress.host is required" .Values.ingress.host | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ include "radiator.fullname" . }} + port: {name: http} +{{- end }} diff --git a/deploy/helm/radiator/templates/postgres.yaml b/deploy/helm/radiator/templates/postgres.yaml new file mode 100644 index 0000000..caca323 --- /dev/null +++ b/deploy/helm/radiator/templates/postgres.yaml @@ -0,0 +1,77 @@ +{{- if .Values.postgres.enabled }} +{{- if not .Values.postgres.existingSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "radiator.postgresSecretName" . }} + labels: {{- include "radiator.labels" . | nindent 4 }} + annotations: + # never delete the database password with the release + helm.sh/resource-policy: keep +type: Opaque +stringData: + password: {{ required "postgres.password is required (or postgres.existingSecret)" .Values.postgres.password | quote }} +--- +{{- end }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "radiator.postgresName" . }} + labels: + {{- include "radiator.labels" . | nindent 4 }} + app.kubernetes.io/component: database +spec: + serviceName: {{ include "radiator.postgresName" . }} + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: database + template: + metadata: + labels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: database + spec: + containers: + - name: postgres + image: {{ .Values.postgres.image }} + ports: + - {name: postgres, containerPort: 5432} + env: + - {name: POSTGRES_USER, value: radiator} + - {name: POSTGRES_DB, value: radiator} + - name: POSTGRES_PASSWORD + valueFrom: {secretKeyRef: {name: {{ include "radiator.postgresSecretName" . }}, key: password}} + - {name: PGDATA, value: /var/lib/postgresql/data/pgdata} + readinessProbe: + exec: {command: ["pg_isready", "-U", "radiator"]} + periodSeconds: 10 + resources: {{- toYaml .Values.postgres.resources | nindent 12 }} + volumeMounts: + - {name: data, mountPath: /var/lib/postgresql/data} + volumeClaimTemplates: + - metadata: {name: data} + spec: + accessModes: [ReadWriteOnce] + {{- with .Values.postgres.storageClassName }} + storageClassName: {{ . }} + {{- end }} + resources: + requests: {storage: {{ .Values.postgres.storage }}} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "radiator.postgresName" . }} + labels: + {{- include "radiator.labels" . | nindent 4 }} + app.kubernetes.io/component: database +spec: + selector: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: database + ports: + - {name: postgres, port: 5432, targetPort: postgres} +{{- end }} diff --git a/deploy/helm/radiator/templates/secret.yaml b/deploy/helm/radiator/templates/secret.yaml new file mode 100644 index 0000000..08f95bf --- /dev/null +++ b/deploy/helm/radiator/templates/secret.yaml @@ -0,0 +1,20 @@ +{{- if not .Values.existingSecret }} +{{- $s := .Values.secrets }} +{{- $url := $s.databaseUrl }} +{{- if and (not $url) .Values.postgres.enabled }} +{{- $pw := required "postgres.password is needed to build the database URL (or set secrets.databaseUrl, or use existingSecret)" .Values.postgres.password }} +{{- $url = printf "postgresql+psycopg://radiator:%s@%s:5432/radiator" ($pw | urlquery) (include "radiator.postgresName" .) }} +{{- end }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "radiator.secretName" . }} + labels: {{- include "radiator.labels" . | nindent 4 }} +type: Opaque +stringData: + database-url: {{ required "secrets.databaseUrl is required (or enable postgres, or use existingSecret)" $url | quote }} + api-token: {{ required "secrets.apiToken is required (or use existingSecret)" $s.apiToken | quote }} + username: {{ required "secrets.username is required (or use existingSecret)" $s.username | quote }} + password: {{ required "secrets.password is required (or use existingSecret)" $s.password | quote }} + session-secret: {{ required "secrets.sessionSecret is required (or use existingSecret)" $s.sessionSecret | quote }} +{{- end }} diff --git a/deploy/helm/radiator/templates/service.yaml b/deploy/helm/radiator/templates/service.yaml new file mode 100644 index 0000000..5ee3760 --- /dev/null +++ b/deploy/helm/radiator/templates/service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "radiator.fullname" . }} + labels: {{- include "radiator.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + selector: {{- include "radiator.selector" . | nindent 4 }} + ports: + - {name: http, port: {{ .Values.service.port }}, targetPort: http} diff --git a/deploy/helm/radiator/templates/zebrunner-import.yaml b/deploy/helm/radiator/templates/zebrunner-import.yaml new file mode 100644 index 0000000..f531994 --- /dev/null +++ b/deploy/helm/radiator/templates/zebrunner-import.yaml @@ -0,0 +1,46 @@ +{{- if .Values.zebrunnerImport.enabled }} +{{- $ze := .Values.zebrunnerImport }} +apiVersion: batch/v1 +kind: Job +metadata: + # one Job per release revision, so enabling it again re-runs the import + # (already-imported runs are skipped) + name: {{ include "radiator.fullname" . }}-zebrunner-import-{{ .Release.Revision }} + labels: {{- include "radiator.labels" . | nindent 4 }} +spec: + backoffLimit: 0 + ttlSecondsAfterFinished: 604800 + template: + spec: + restartPolicy: Never + {{- with .Values.imagePullSecrets }} + imagePullSecrets: {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: import + image: {{ include "radiator.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["python", "-m", "radiator.zebrunner_import"] + args: + - --zebrunner-url=$(ZE_BASE_URL) + - --refresh-token=$(ZE_REFRESH_TOKEN) + - --radiator-url=http://{{ include "radiator.fullname" . }}:{{ .Values.service.port }} + - --radiator-token=$(RADIATOR_TOKEN) + {{- with $ze.projectId }} + - --project-id={{ . }} + {{- end }} + {{- with $ze.env }} + - --env={{ . }} + {{- end }} + {{- range $ze.args }} + - {{ . | quote }} + {{- end }} + env: + - name: ZE_BASE_URL + valueFrom: {secretKeyRef: {name: {{ $ze.existingSecret }}, key: ze-base-url}} + - name: ZE_REFRESH_TOKEN + valueFrom: {secretKeyRef: {name: {{ $ze.existingSecret }}, key: ze-refresh-token}} + - name: RADIATOR_TOKEN + valueFrom: {secretKeyRef: {name: {{ include "radiator.secretName" . }}, key: api-token}} + resources: {{- toYaml $ze.resources | nindent 12 }} +{{- end }} diff --git a/deploy/helm/radiator/values.example.yaml b/deploy/helm/radiator/values.example.yaml new file mode 100644 index 0000000..524dfe8 --- /dev/null +++ b/deploy/helm/radiator/values.example.yaml @@ -0,0 +1,49 @@ +# Example values for the radiator chart. Copy to values..yaml next to +# this file (those are git-ignored), fill it in, and install with +# +# helm upgrade --install radiator deploy/helm/radiator \ +# -n translator -f deploy/helm/radiator/values..yaml +# +# Keep real secrets out of git: either create the Secret yourself (below, +# recommended) or keep the copy of this file outside the repo. + +image: + # Pin a release rather than latest, eg the tag the release workflow pushed. + tag: latest + +# Created with: +# kubectl -n translator create secret generic radiator-secrets \ +# --from-literal=database-url='postgresql+psycopg://radiator:PASSWORD@radiator-postgres:5432/radiator' \ +# --from-literal=api-token="$(openssl rand -base64 32)" \ +# --from-literal=username=translator \ +# --from-literal=password="$(openssl rand -base64 18)" \ +# --from-literal=session-secret="$(openssl rand -base64 32)" +existingSecret: radiator-secrets + +# ...or let the chart create it (then leave existingSecret empty): +# secrets: +# databaseUrl: "" # empty: the chart's own Postgres, if enabled +# apiToken: CHANGE-ME +# username: translator +# password: CHANGE-ME +# sessionSecret: CHANGE-ME + +ingress: + enabled: true + className: nginx + host: radiator.example.org + tlsSecretName: radiator-tls + +# Only if the namespace has no database to use. +postgres: + enabled: true + # kubectl -n translator create secret generic radiator-postgres \ + # --from-literal=password="$(openssl rand -base64 24)" + existingSecret: radiator-postgres + storage: 10Gi + # storageClassName: standard + +# To bring over Zebrunner's history once the radiator is up: +# zebrunnerImport: +# enabled: true +# args: ["--since", "2026-09-15", "--dry-run"] diff --git a/deploy/helm/radiator/values.yaml b/deploy/helm/radiator/values.yaml new file mode 100644 index 0000000..c779912 --- /dev/null +++ b/deploy/helm/radiator/values.yaml @@ -0,0 +1,103 @@ +# Defaults for the radiator chart. Don't edit this file for a deployment: +# copy values.example.yaml to values..yaml (git-ignored) and pass that +# with -f. See deploy/helm/README.md. + +image: + repository: ghcr.io/translatorsri/testharness-radiator + # Defaults to the chart's appVersion (latest). Pin a release for production. + tag: "" + pullPolicy: IfNotPresent +imagePullSecrets: [] + +# The radiator's secrets. Either name a Secret you've created yourself, with +# the keys database-url, api-token, username, password and session-secret, or +# leave existingSecret empty and set them under `secrets` for the chart to +# create one. +existingSecret: "" +secrets: + # postgresql+psycopg://USER:PASSWORD@HOST:5432/DB. Leave empty with + # postgres.enabled and the chart points it at its own Postgres. + databaseUrl: "" + # The bearer token the harness uploads with (its RADIATOR_TOKEN). + apiToken: "" + # The one shared login for the consortium. + username: translator + password: "" + # Signs the session cookie; changing it logs everyone out. + sessionSecret: "" + +config: + # Only send the session cookie over HTTPS. Turn off only for plain-http + # testing. + secureCookies: true + # Environments the run grid greys out (comma-separated). + excludedEnvs: dev + # The radiator's own log lines: each upload, and why one was rejected. + logLevel: INFO + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: nginx + host: radiator.example.org + # An existing TLS secret for the host; empty for no TLS. + tlsSecretName: "" + annotations: + # uploads from the harness come in batches; leave room for big ones + nginx.ingress.kubernetes.io/proxy-body-size: "16m" + +resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {memory: 512Mi} + +# The migration init container, run before every start; a no-op when the +# schema is current. +migrate: + resources: + requests: {cpu: 50m, memory: 128Mi} + limits: {memory: 256Mi} + +podSecurityContext: + runAsNonRoot: true +nodeSelector: {} +tolerations: [] +affinity: {} + +# A small in-namespace Postgres, for when there's no database to use. Back +# its volume up: it's the only copy of the results history. +postgres: + enabled: false + image: postgres:16 + # A Secret with the key `password`; or leave empty and set `password` for + # the chart to create one. + existingSecret: "" + password: "" + storage: 10Gi + storageClassName: "" + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {memory: 1Gi} + +# A one-off Job importing past runs from Zebrunner (see +# docs/information-radiator.md). Enable for one install/upgrade, then turn off. +zebrunnerImport: + enabled: false + # A Secret with the keys ze-base-url and ze-refresh-token (the harness's + # test-harness-secrets has them). + existingSecret: test-harness-secrets + # Zebrunner's project id (the projectId in the API calls its launches page + # makes): newer Zebrunners need it; older ones (eg 1.28) don't. + projectId: "" + # Zebrunner never recorded the environment. Empty: each run's is looked up + # from its pks on each environment's ARS. Set it to record every imported + # run under one environment instead. + env: "" + # Extra arguments, eg ["--dry-run", "--limit", "5"], ["--since", + # "2026-09-15"] or ["--ars", "ci=https://..."]. + args: [] + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {memory: 1Gi} diff --git a/deploy/helm/test-harness/.helmignore b/deploy/helm/test-harness/.helmignore new file mode 100644 index 0000000..bca6ed2 --- /dev/null +++ b/deploy/helm/test-harness/.helmignore @@ -0,0 +1,2 @@ +# copies of values.example.yaml stay out of the chart package +values.*.yaml diff --git a/deploy/helm/test-harness/Chart.yaml b/deploy/helm/test-harness/Chart.yaml new file mode 100644 index 0000000..ea63388 --- /dev/null +++ b/deploy/helm/test-harness/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: test-harness +description: Scheduled Translator Test Harness runs - weekly run cycles per environment, the Monday digest, and performance sweeps +type: application +version: 0.1.0 +# The image tag when values don't set one. +appVersion: "latest" diff --git a/deploy/helm/test-harness/templates/NOTES.txt b/deploy/helm/test-harness/templates/NOTES.txt new file mode 100644 index 0000000..4e5e14b --- /dev/null +++ b/deploy/helm/test-harness/templates/NOTES.txt @@ -0,0 +1,14 @@ +{{- $cycles := .Values.cycles.enabled | ternary .Values.cycles.environments list }} +Scheduled Test Harness runs ({{ .Values.timeZone }}): +{{- range $cycles }} + {{ include "harness.fullname" $ }}-cycle-{{ .name }} "{{ .schedule }}"{{ if .suspend }} (suspended){{ end }} +{{- end }} +{{- if .Values.digest.enabled }} + {{ include "harness.fullname" . }}-weekly-digest "{{ .Values.digest.schedule }}" +{{- end }} +{{- if .Values.sweep.enabled }} + {{ include "harness.fullname" . }}-performance-sweep "{{ .Values.sweep.schedule }}" +{{- end }} + +Run one now, eg: + kubectl -n {{ .Release.Namespace }} create job --from=cronjob/{{ include "harness.fullname" . }}-weekly-digest digest-now diff --git a/deploy/helm/test-harness/templates/_helpers.tpl b/deploy/helm/test-harness/templates/_helpers.tpl new file mode 100644 index 0000000..87f34a6 --- /dev/null +++ b/deploy/helm/test-harness/templates/_helpers.tpl @@ -0,0 +1,80 @@ +{{- define "harness.fullname" -}} +{{- if contains .Chart.Name .Release.Name -}} +{{- .Release.Name | trunc 40 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name .Chart.Name | trunc 40 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{- define "harness.labels" -}} +app.kubernetes.io/name: {{ .Chart.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version }} +{{- end -}} + +{{- define "harness.image" -}} +{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }} +{{- end -}} + +{{- define "harness.secretName" -}} +{{ .Values.existingSecret | default (printf "%s-secrets" (include "harness.fullname" .)) }} +{{- end -}} + +{{/* Where the suites come from: before any `--`. */}} +{{- define "harness.testsArgs" -}} +{{- if .Values.tests.testsUrl }} +- --tests_url +- {{ .Values.tests.testsUrl | quote }} +{{- else if .Values.tests.download }} +- --download +{{- end }} +{{- end -}} + +{{- define "harness.env" -}} +- name: PYTHONUNBUFFERED + value: "1" +- name: RADIATOR_URL + value: {{ required "radiator.url is required" .Values.radiator.url | quote }} +{{- with .Values.radiator.publicUrl }} +- name: RADIATOR_PUBLIC_URL + value: {{ . | quote }} +{{- end }} +- name: RADIATOR_TOKEN + valueFrom: {secretKeyRef: {name: {{ include "harness.secretName" . }}, key: radiator-token}} +{{- range $env, $key := dict "SLACK_WEBHOOK_URL" "slack-webhook-url" "SLACK_TOKEN" "slack-token" "SLACK_CHANNEL" "slack-channel" "RADIATOR_SLACK_WEBHOOK_URL" "radiator-slack-webhook-url" "RADIATOR_SLACK_CHANNEL" "radiator-slack-channel" "RADIATOR_SLACK_TOKEN" "radiator-slack-token" "ZE_BASE_URL" "ze-base-url" "ZE_REFRESH_TOKEN" "ze-refresh-token" }} +- name: {{ $env }} + valueFrom: {secretKeyRef: {name: {{ include "harness.secretName" $ }}, key: {{ $key }}, optional: true}} +{{- end }} +{{- with .Values.queryParameters }} +- name: QUERY_PARAMETERS + value: {{ . | quote }} +{{- end }} +{{- end -}} + +{{/* The pod spec around one container; pass (dict "root" $ "container" ...). */}} +{{- define "harness.podSpec" -}} +{{- $ := .root -}} +restartPolicy: Never +{{- with $.Values.imagePullSecrets }} +imagePullSecrets: {{- toYaml . | nindent 2 }} +{{- end }} +{{- with $.Values.podSecurityContext }} +securityContext: {{- toYaml . | nindent 2 }} +{{- end }} +{{- with $.Values.nodeSelector }} +nodeSelector: {{- toYaml . | nindent 2 }} +{{- end }} +{{- with $.Values.tolerations }} +tolerations: {{- toYaml . | nindent 2 }} +{{- end }} +{{- with $.Values.affinity }} +affinity: {{- toYaml . | nindent 2 }} +{{- end }} +containers: + {{- .container | nindent 2 }} +volumes: + # results land here; what matters is uploaded to the radiator and Slack + - {name: results, emptyDir: {}} +{{- end -}} diff --git a/deploy/helm/test-harness/templates/cycles.yaml b/deploy/helm/test-harness/templates/cycles.yaml new file mode 100644 index 0000000..363cea2 --- /dev/null +++ b/deploy/helm/test-harness/templates/cycles.yaml @@ -0,0 +1,90 @@ +{{- if .Values.cycles.enabled }} +{{- $d := .Values.cycles.defaults }} +{{- range $env := .Values.cycles.environments }} +{{- $name := required "each cycles.environments entry needs a name" $env.name }} +{{- $pick := dict }} +{{- range $field := list "acceptance" "pathfinder" "performance" "performanceProfile" "activeDeadlineSeconds" "resources" }} +{{- $_ := set $pick $field (ternary (get $env $field) (get $d $field) (hasKey $env $field)) }} +{{- end }} +{{- $targets := $env.performanceTargets | default list }} +{{- if and $pick.performance (not $targets) }} +{{- fail (printf "cycles.environments[%s]: performance needs performanceTargets, eg [\"ars=https://ars.%s.transltr.io\"] (or set performance: \"\")" $name $name) }} +{{- end }} +{{- if not (or $pick.acceptance $pick.pathfinder $pick.performance) }} +{{- fail (printf "cycles.environments[%s]: nothing to run" $name) }} +{{- end }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "harness.fullname" $ }}-cycle-{{ $name }} + labels: + {{- include "harness.labels" $ | nindent 4 }} + app.kubernetes.io/component: cycle + translator.ncats.io/environment: {{ $name | quote }} +spec: + schedule: {{ required (printf "cycles.environments[%s].schedule is required" $name) $env.schedule | quote }} + timeZone: {{ $.Values.timeZone | quote }} + suspend: {{ $env.suspend | default false }} + # Never two cycles at once: they would load the same services together. + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + # Never retry: a retry re-runs every step and re-posts every report. + # The Job only fails when a step couldn't run at all. + backoffLimit: 0 + activeDeadlineSeconds: {{ $pick.activeDeadlineSeconds }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ $.Chart.Name }} + app.kubernetes.io/instance: {{ $.Release.Name }} + app.kubernetes.io/component: cycle + spec: + terminationGracePeriodSeconds: 120 + {{- $container := include "harness.cycleContainer" (dict "root" $ "env" $env "pick" $pick "targets" $targets) }} + {{- include "harness.podSpec" (dict "root" $ "container" $container) | nindent 10 }} +{{- end }} +{{- end }} + +{{- define "harness.cycleContainer" -}} +{{- $ := .root -}} +- name: test-harness + image: {{ include "harness.image" $ }} + imagePullPolicy: {{ $.Values.image.pullPolicy }} + command: ["test-harness-cycle"] + args: + {{- include "harness.testsArgs" $ | trim | nindent 4 }} + {{- with .pick.acceptance }} + - --acceptance + - {{ . | quote }} + {{- end }} + {{- with .pick.pathfinder }} + - --pathfinder + - {{ . | quote }} + {{- end }} + {{- if .pick.performance }} + - --performance + - {{ .pick.performance | quote }} + {{- range .targets }} + - --performance_target + - {{ . | quote }} + {{- end }} + {{- with .pick.performanceProfile }} + - --performance_profile + - {{ . | quote }} + {{- end }} + {{- end }} + # everything after `--` goes to every step + - "--" + {{- range (concat $.Values.harnessArgs (.env.harnessArgs | default list)) }} + - {{ . | quote }} + {{- end }} + env: + {{- include "harness.env" $ | nindent 4 }} + resources: {{- toYaml .pick.resources | nindent 4 }} + volumeMounts: + - {name: results, mountPath: /app/test_results} +{{- end -}} diff --git a/deploy/helm/test-harness/templates/digest.yaml b/deploy/helm/test-harness/templates/digest.yaml new file mode 100644 index 0000000..7e89062 --- /dev/null +++ b/deploy/helm/test-harness/templates/digest.yaml @@ -0,0 +1,40 @@ +{{- if .Values.digest.enabled }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "harness.fullname" . }}-weekly-digest + labels: + {{- include "harness.labels" . | nindent 4 }} + app.kubernetes.io/component: digest +spec: + schedule: {{ .Values.digest.schedule | quote }} + timeZone: {{ .Values.timeZone | quote }} + suspend: {{ .Values.digest.suspend }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + # it only reads from the radiator, so one retry is harmless + backoffLimit: 1 + activeDeadlineSeconds: 600 + template: + metadata: + labels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: digest + spec: + {{- $container := include "harness.digestContainer" . }} + {{- include "harness.podSpec" (dict "root" . "container" $container) | nindent 10 }} +{{- end }} + +{{- define "harness.digestContainer" -}} +- name: digest + image: {{ include "harness.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["test-harness-radiator", "digest", "--days", {{ .Values.digest.days | quote }}] + env: + {{- include "harness.env" . | nindent 4 }} + resources: {{- toYaml .Values.digest.resources | nindent 4 }} +{{- end -}} diff --git a/deploy/helm/test-harness/templates/secret.yaml b/deploy/helm/test-harness/templates/secret.yaml new file mode 100644 index 0000000..a97a3fc --- /dev/null +++ b/deploy/helm/test-harness/templates/secret.yaml @@ -0,0 +1,16 @@ +{{- if not .Values.existingSecret }} +{{- $s := .Values.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "harness.secretName" . }} + labels: {{- include "harness.labels" . | nindent 4 }} +type: Opaque +stringData: + radiator-token: {{ required "secrets.radiatorToken is required (or use existingSecret)" $s.radiatorToken | quote }} + {{- range $key, $value := dict "slack-webhook-url" $s.slackWebhookUrl "slack-token" $s.slackToken "slack-channel" $s.slackChannel "radiator-slack-webhook-url" $s.radiatorSlackWebhookUrl "radiator-slack-channel" $s.radiatorSlackChannel "radiator-slack-token" $s.radiatorSlackToken "ze-base-url" $s.zeBaseUrl "ze-refresh-token" $s.zeRefreshToken }} + {{- if $value }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/deploy/helm/test-harness/templates/sweep.yaml b/deploy/helm/test-harness/templates/sweep.yaml new file mode 100644 index 0000000..b18041a --- /dev/null +++ b/deploy/helm/test-harness/templates/sweep.yaml @@ -0,0 +1,61 @@ +{{- if .Values.sweep.enabled }} +{{- if not .Values.sweep.targets }} +{{- fail "sweep.targets is required when the sweep is enabled, eg [\"aragorn=https://aragorn.ci.transltr.io\"]" }} +{{- end }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "harness.fullname" . }}-performance-sweep + labels: + {{- include "harness.labels" . | nindent 4 }} + app.kubernetes.io/component: sweep +spec: + schedule: {{ .Values.sweep.schedule | quote }} + timeZone: {{ .Values.timeZone | quote }} + suspend: {{ .Values.sweep.suspend }} + # A sweep runs for hours; never start a second one on top of it. + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 0 + activeDeadlineSeconds: {{ .Values.sweep.activeDeadlineSeconds }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: sweep + spec: + terminationGracePeriodSeconds: 120 + {{- $container := include "harness.sweepContainer" . }} + {{- include "harness.podSpec" (dict "root" . "container" $container) | nindent 10 }} +{{- end }} + +{{- define "harness.sweepContainer" -}} +- name: test-harness + image: {{ include "harness.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["test-harness-sweep"] + args: + - --suite + - {{ .Values.sweep.suite | quote }} + {{- include "harness.testsArgs" . | trim | nindent 4 }} + {{- range .Values.sweep.targets }} + - {{ . | quote }} + {{- end }} + - "--" + {{- with .Values.sweep.profile }} + - --performance_profile + - {{ . | quote }} + {{- end }} + {{- range .Values.harnessArgs }} + - {{ . | quote }} + {{- end }} + env: + {{- include "harness.env" . | nindent 4 }} + resources: {{- toYaml .Values.sweep.resources | nindent 4 }} + volumeMounts: + - {name: results, mountPath: /app/test_results} +{{- end -}} diff --git a/deploy/helm/test-harness/values.example.yaml b/deploy/helm/test-harness/values.example.yaml new file mode 100644 index 0000000..a3fa438 --- /dev/null +++ b/deploy/helm/test-harness/values.example.yaml @@ -0,0 +1,91 @@ +# Example values for the test-harness chart. Copy to values..yaml next +# to this file (those are git-ignored), fill it in, and install with +# +# helm upgrade --install test-harness deploy/helm/test-harness \ +# -n translator -f deploy/helm/test-harness/values..yaml +# +# Keep real secrets out of git: either create the Secret yourself (below, +# recommended) or keep the copy of this file outside the repo. + +image: + # Pin a release rather than latest, eg the tag the release workflow pushed. + tag: latest + +# Created with: +# kubectl -n translator create secret generic test-harness-secrets \ +# --from-literal=radiator-token='' \ +# --from-literal=slack-webhook-url='https://hooks.slack.com/services/...' \ +# --from-literal=slack-token='xoxb-...' \ +# --from-literal=slack-channel='C0123ABCD' +# +# Alongside Zebrunner, to give the new radiator a channel of its own (the +# report with the Zebrunner link stays in slack-channel), add: +# --from-literal=radiator-slack-webhook-url='https://hooks.slack.com/services/...' \ +# --from-literal=radiator-slack-channel='C0456EFGH' +# and, for Zebrunner itself: +# --from-literal=ze-base-url='https://zebrunner.example.org' \ +# --from-literal=ze-refresh-token='...' +existingSecret: test-harness-secrets + +# ...or let the chart create it (then leave existingSecret empty): +# secrets: +# radiatorToken: CHANGE-ME +# slackWebhookUrl: https://hooks.slack.com/services/your-webhook-path +# slackToken: xoxb-your-bot-token +# slackChannel: C0123ABCD +# radiatorSlackWebhookUrl: https://hooks.slack.com/services/another-webhook-path +# radiatorSlackChannel: C0456EFGH + +radiator: + # The radiator chart's service, in the same namespace. + url: http://radiator + publicUrl: https://radiator.example.org + +tests: + download: true + # testsUrl: https://github.com/NCATSTranslator/Tests/archive/refs/heads/main.zip + +harnessArgs: ["--log_level", "INFO"] +# queryParameters: '{"timeout": 300}' + +# One environment a day, Monday to Thursday, so a regression shows up in dev +# before it reaches prod; Friday is free for re-runs. The environment comes +# from the test cases, so each environment names the suites written for it. +cycles: + enabled: true + defaults: + pathfinder: pathfinder_tests + performance: performance_tests + performanceProfile: mixed + environments: + - name: dev + schedule: "52 1 * * 1" + acceptance: sprint_4_tests + performanceTargets: ["ars=https://ars.dev.transltr.io"] + - name: ci + schedule: "52 1 * * 2" + acceptance: sprint_4_tests + performanceTargets: ["ars=https://ars.ci.transltr.io"] + - name: test + schedule: "52 1 * * 3" + acceptance: sprint_4_tests + performanceTargets: ["ars=https://ars.test.transltr.io"] + - name: prod + schedule: "52 1 * * 4" + acceptance: sprint_4_tests + performanceTargets: ["ars=https://ars.transltr.io"] + # eg to skip the load test on prod: + # performance: "" + +digest: + enabled: true + schedule: "52 8 * * 1" + +# The ARAs on ci, every Sunday, outside the cycles. +sweep: + enabled: false + schedule: "0 2 * * 0" + targets: + - aragorn=https://aragorn.ci.transltr.io + - arax=https://arax.ci.transltr.io + - bte=https://bte.ci.transltr.io diff --git a/deploy/helm/test-harness/values.yaml b/deploy/helm/test-harness/values.yaml new file mode 100644 index 0000000..5d0baf7 --- /dev/null +++ b/deploy/helm/test-harness/values.yaml @@ -0,0 +1,115 @@ +# Defaults for the test-harness chart. Don't edit this file for a deployment: +# copy values.example.yaml to values..yaml (git-ignored) and pass that +# with -f. See deploy/helm/README.md. + +image: + repository: ghcr.io/translatorsri/testharness + # Defaults to the chart's appVersion (latest). Pin a release for production. + tag: "" + pullPolicy: IfNotPresent +imagePullSecrets: [] + +# Schedules are in this time zone (needs Kubernetes >= 1.27). +timeZone: America/New_York + +# The harness's secrets. Either name a Secret you've created yourself, with +# the keys below, or leave existingSecret empty and set them under `secrets` +# for the chart to create one. +# radiator-token the radiator's api-token +# slack-webhook-url, slack-token, slack-channel optional: without them, +# reports are saved to files +# radiator-slack-webhook-url, radiator-slack-channel +# optional: a channel of the +# radiator's own (below) +# radiator-slack-token optional: defaults to +# slack-token +# ze-base-url, ze-refresh-token optional: Zebrunner, until +# the radiator takes over +existingSecret: "" +secrets: + radiatorToken: "" + slackWebhookUrl: "" + slackToken: "" + # The channel ID (eg C0123ABCD), not its #name. + slackChannel: "" + # The new radiator's own channel, while it runs alongside Zebrunner. With + # Zebrunner configured, SLACK_CHANNEL is Zebrunner's and gets only its + # report; the radiator's reports, the cycle boards and the digest go here + # (or nowhere without these). Its own incoming webhook; the same bot by + # default. + radiatorSlackWebhookUrl: "" + radiatorSlackChannel: "" + radiatorSlackToken: "" + zeBaseUrl: "" + zeRefreshToken: "" + +radiator: + # Where the harness uploads: the radiator's in-cluster service. + url: http://radiator + # Where people open it, for the links in Slack. + publicUrl: "" + +tests: + # Download the suites from the Tests repo (otherwise they must be in the + # image). + download: true + # Pin a branch or fork, eg + # https://github.com/NCATSTranslator/Tests/archive/refs/heads/main.zip + testsUrl: "" + +# Passed to every harness run (after the `--`). +harnessArgs: ["--log_level", "INFO"] +# JSON sent as parameters with every query, eg '{"timeout": 300}'. +queryParameters: "" + +podSecurityContext: {} +nodeSelector: {} +tolerations: [] +affinity: {} + +# Weekly run cycles: acceptance, then pathfinder, then performance, in one +# environment. One CronJob per entry in `environments`. +cycles: + enabled: true + # Applied to every environment unless the environment sets its own. + defaults: + pathfinder: pathfinder_tests + performance: performance_tests + performanceProfile: mixed + # ~2 h expected; this stops a wedged run before the next environment's day. + activeDeadlineSeconds: 14400 + resources: + # HelmsDeep's load generator runs in this container during the + # performance step: give it real CPU, requests == limits. + requests: {cpu: "2", memory: "2Gi"} + limits: {cpu: "2", memory: "4Gi"} + # Each: name, schedule, acceptance, performanceTargets (NAME=URL, in + # order), and optionally pathfinder, performance, performanceProfile, + # harnessArgs, suspend, activeDeadlineSeconds, resources. See + # values.example.yaml. + environments: [] + +# The Monday digest: every environment's latest runs, posted to Slack. +digest: + enabled: true + schedule: "52 8 * * 1" + days: 7 + suspend: false + resources: + requests: {cpu: 100m, memory: 128Mi} + limits: {memory: 256Mi} + +# A performance sweep of several services in turn (test-harness-sweep), eg +# the ARAs, outside the cycles. ~61 min per ARA with the mixed profile. +sweep: + enabled: false + schedule: "0 2 * * 0" + suspend: false + suite: performance_tests + profile: mixed + # NAME=URL, in the order they're loaded. + targets: [] + activeDeadlineSeconds: 21600 + resources: + requests: {cpu: "2", memory: "2Gi"} + limits: {cpu: "2", memory: "4Gi"} diff --git a/deploy/radiator-import.example.yaml b/deploy/radiator-import.example.yaml new file mode 100644 index 0000000..e0697a2 --- /dev/null +++ b/deploy/radiator-import.example.yaml @@ -0,0 +1,43 @@ +# The Helm charts in deploy/helm/ are the maintained way to deploy; these plain +# manifests are kept as a reference for clusters without Helm. +# +# One-off Job: import past runs from Zebrunner into the Information Radiator. +# Run it once the radiator is up; it's safe to re-run (already-imported runs +# are skipped). Try `--dry-run` and `--limit 5` first to check what comes +# across. +# +# Zebrunner never recorded the environment, so each run's is looked up from its +# pks on each environment's ARS (see --ars, --no-env-lookup); --env records +# every run under one environment instead. +apiVersion: batch/v1 +kind: Job +metadata: + name: radiator-zebrunner-import +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + containers: + - name: import + image: ghcr.io/translatorsri/testharness-radiator:latest + command: ["python", "-m", "radiator.zebrunner_import"] + args: + - --zebrunner-url=$(ZE_BASE_URL) + - --refresh-token=$(ZE_REFRESH_TOKEN) + - --radiator-url=http://radiator + - --radiator-token=$(RADIATOR_TOKEN) + # - --project-id=1 # newer Zebrunners (the v1 API) need it + # without --env, each run's environment is looked up from its pks + # on each environment's ARS + - --since=2026-09-15 + env: + - name: ZE_BASE_URL + valueFrom: {secretKeyRef: {name: test-harness-secrets, key: ze-base-url}} + - name: ZE_REFRESH_TOKEN + valueFrom: {secretKeyRef: {name: test-harness-secrets, key: ze-refresh-token}} + - name: RADIATOR_TOKEN + valueFrom: {secretKeyRef: {name: radiator-secrets, key: api-token}} + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {memory: 1Gi} diff --git a/deploy/radiator-postgres.example.yaml b/deploy/radiator-postgres.example.yaml new file mode 100644 index 0000000..c09c43a --- /dev/null +++ b/deploy/radiator-postgres.example.yaml @@ -0,0 +1,62 @@ +# The Helm charts in deploy/helm/ are the maintained way to deploy; these plain +# manifests are kept as a reference for clusters without Helm. +# +# A small Postgres for the Information Radiator, for when the namespace has no +# database to use. Skip this if you have a managed one. +# +# kubectl create secret generic radiator-postgres \ +# --from-literal=password="$(openssl rand -base64 24)" +# +# and use that password in radiator-secrets' database-url: +# postgresql+psycopg://radiator:@radiator-postgres:5432/radiator +# +# Back the volume up: it is the only copy of the results history. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: radiator-postgres + labels: {app: radiator-postgres} +spec: + serviceName: radiator-postgres + replicas: 1 + selector: + matchLabels: {app: radiator-postgres} + template: + metadata: + labels: {app: radiator-postgres} + spec: + containers: + - name: postgres + image: postgres:16 + ports: + - {name: postgres, containerPort: 5432} + env: + - {name: POSTGRES_USER, value: radiator} + - {name: POSTGRES_DB, value: radiator} + - name: POSTGRES_PASSWORD + valueFrom: {secretKeyRef: {name: radiator-postgres, key: password}} + - {name: PGDATA, value: /var/lib/postgresql/data/pgdata} + readinessProbe: + exec: {command: ["pg_isready", "-U", "radiator"]} + periodSeconds: 10 + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {memory: 1Gi} + volumeMounts: + - {name: data, mountPath: /var/lib/postgresql/data} + volumeClaimTemplates: + - metadata: {name: data} + spec: + accessModes: [ReadWriteOnce] + resources: + requests: {storage: 10Gi} +--- +apiVersion: v1 +kind: Service +metadata: + name: radiator-postgres + labels: {app: radiator-postgres} +spec: + selector: {app: radiator-postgres} + ports: + - {name: postgres, port: 5432, targetPort: postgres} diff --git a/deploy/radiator.example.yaml b/deploy/radiator.example.yaml new file mode 100644 index 0000000..6703101 --- /dev/null +++ b/deploy/radiator.example.yaml @@ -0,0 +1,111 @@ +# The Helm charts in deploy/helm/ are the maintained way to deploy; these plain +# manifests are kept as a reference for clusters without Helm. +# +# Example manifests for the Information Radiator, in the same namespace as the +# harness CronJob. Adapt before applying: the image tag, hostname, TLS secret, +# ingress class, and every value in the Secret are placeholders. +# +# The radiator needs a Postgres database. Point RADIATOR_DATABASE_URL at an +# existing one, or apply deploy/radiator-postgres.example.yaml for a small +# in-namespace instance. +# +# Generate the secrets with eg `openssl rand -base64 32`: +# kubectl create secret generic radiator-secrets \ +# --from-literal=database-url='postgresql+psycopg://radiator:PASSWORD@radiator-postgres:5432/radiator' \ +# --from-literal=api-token="$(openssl rand -base64 32)" \ +# --from-literal=username=translator \ +# --from-literal=password="$(openssl rand -base64 18)" \ +# --from-literal=session-secret="$(openssl rand -base64 32)" +# +# Then give the harness the same api-token as RADIATOR_TOKEN (see +# deploy/cronjob.example.yaml), and share the username/password with the +# consortium. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: radiator + labels: {app: radiator} +spec: + # One replica: the login throttle is in memory, and the load is a handful + # of people looking at a dashboard. Recreate so a migration never runs + # against a database an old pod is still using. + replicas: 1 + strategy: {type: Recreate} + selector: + matchLabels: {app: radiator} + template: + metadata: + labels: {app: radiator} + spec: + securityContext: + runAsNonRoot: true + initContainers: + # Bring the schema up to date before the app starts. Alembic is a + # no-op when it already is. + - name: migrate + image: ghcr.io/translatorsri/testharness-radiator:latest + command: ["alembic", "-c", "radiator/alembic.ini", "upgrade", "head"] + env: + - name: RADIATOR_DATABASE_URL + valueFrom: {secretKeyRef: {name: radiator-secrets, key: database-url}} + resources: + requests: {cpu: 50m, memory: 128Mi} + limits: {memory: 256Mi} + containers: + - name: radiator + image: ghcr.io/translatorsri/testharness-radiator:latest + ports: + - {name: http, containerPort: 8000} + env: + - name: RADIATOR_DATABASE_URL + valueFrom: {secretKeyRef: {name: radiator-secrets, key: database-url}} + - name: RADIATOR_API_TOKEN + valueFrom: {secretKeyRef: {name: radiator-secrets, key: api-token}} + - name: RADIATOR_USERNAME + valueFrom: {secretKeyRef: {name: radiator-secrets, key: username}} + - name: RADIATOR_PASSWORD + valueFrom: {secretKeyRef: {name: radiator-secrets, key: password}} + - name: RADIATOR_SESSION_SECRET + valueFrom: {secretKeyRef: {name: radiator-secrets, key: session-secret}} + readinessProbe: + httpGet: {path: /healthz, port: http} + periodSeconds: 10 + livenessProbe: + # generous: a database blip shouldn't restart the app + httpGet: {path: /healthz, port: http} + periodSeconds: 30 + failureThreshold: 6 + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {memory: 512Mi} +--- +apiVersion: v1 +kind: Service +metadata: + name: radiator + labels: {app: radiator} +spec: + selector: {app: radiator} + ports: + - {name: http, port: 80, targetPort: http} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: radiator + annotations: + # uploads from the harness come in batches; leave room for big ones + nginx.ingress.kubernetes.io/proxy-body-size: "16m" +spec: + ingressClassName: nginx + tls: + - hosts: [radiator.example.org] + secretName: radiator-tls + rules: + - host: radiator.example.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: {name: radiator, port: {name: http}} diff --git a/docs/information-radiator.md b/docs/information-radiator.md new file mode 100644 index 0000000..96539fa --- /dev/null +++ b/docs/information-radiator.md @@ -0,0 +1,336 @@ +# Information Radiator + +A replacement for the Zebrunner-based Information Radiator, built in this repo +alongside the harness. The harness posts structured results to it; it shows +them to the consortium behind one shared login. + +## Why + +Zebrunner's model forced the harness to put its results into labels and log +lines: per-agent status went in as labels, and the full report (found, rank, +score, pks) as a JSON string in a log. None of it could be filtered or charted, +performance results never reached it, and each test cost about five API calls. +The radiator stores results as records, so it can show what changed between +runs and how assets and agents trend over time. + +## Decisions + +| | | +|---|---| +| Repo | Monorepo: `radiator/` here, shipped as its own image (`ghcr.io/translatorsri/testharness-radiator`). | +| Hosting | Same Kubernetes namespace as the harness CronJob. | +| UI login | One shared username/password for the consortium (a k8s secret). Signed session cookie; failed logins are throttled per client. | +| API auth | Every API route needs the bearer token (`RADIATOR_API_TOKEN`); the read API also accepts a logged-in session. Uploads need the token. | +| Raw responses | Not stored. Results keep their pks, which open in the ARAX UI on ci. | +| History | Imported from Zebrunner's API (`radiator.zebrunner_import`). | + +## What's where + +``` +radiator_schema/ the API contract (Pydantic), shared by both sides +test_harness/ + radiator_client.py harness side: batched uploads, save + `test-harness-radiator push`, + and `test-harness-radiator digest` + cycle.py `test-harness-cycle`: a run cycle and its Slack board +radiator/ + app.py, auth.py FastAPI app, login, token auth + api.py, ingest.py ingest + read API; upserts + models.py, migrations/ Postgres schema (SQLAlchemy, Alembic) + queries.py, web.py the dashboard's queries and pages + templates/, static/ server-rendered HTML, CSS, a small hover script + charts.py, grid.py server-side SVG charts, the run grid + history_png.py the pass-rate and concurrency PNGs for Slack + cycles.py, board_png.py run cycles: the board, flags, the weekly digest (and its PNGs) + zebrunner_import.py the history import + demo.py made-up data for development +deploy/ + helm/ Helm charts for the radiator and the harness + radiator.example.yaml Deployment (+ migration init container), Service, Ingress + radiator-postgres.example.yaml a small in-namespace Postgres, if you need one + radiator-import.example.yaml one-off Job for the Zebrunner import + cycle.example.yaml weekly run cycle CronJob, one per environment + digest.example.yaml Monday digest CronJob +``` + +## The dashboard + +- **Weekly**: each environment's latest full runs of acceptance, pathfinder + and performance over a week, against the run before, with a verdict + (regressed, improved, mixed, steady) and the flags; step back a week at a + time. Links to each run cycle. +- **A run cycle**: the board for one environment's cycle (each test against + its previous run, with a step line of its last 8 runs), the flags, and the + regressions and fixes by agent. +- **Live updates**: while a run is going, its page fills in by itself as the + harness uploads each test case's results, with a progress bar (assets tested + out of the run's total) and "so far" on the numbers. The runs list and the + Weekly page pick up runs starting, uploading and finishing. Pages poll a + small JSON check (`/api/runs/{id}/progress`, `/api/live`) every 10 to 60 + seconds, pause while the tab is hidden, and refetch only when something + changed. An open run that hasn't uploaded anything for 2 hours is marked + *didn't finish* and no longer polled. Performance runs have nothing to show + until HelmsDeep finishes. +- **Runs**: every run, newest first, with its status breakdown, pass rate, and + duration; filter by suite and environment. +- **A run**: pass rate against the previous run, regressions and fixes, a + per-agent breakdown (also split by expected output), and the results grid: + every asset against every agent, with the rank the expected answer came back + at. Filter by status (overall or + per agent), expected output, or search by name/id/CURIE. Each cell opens its + pk in ARAX. +- **What changed**: against the previous run *of the same series* (same suite, + env, target override, and query type): regressions, fixes, other status + changes, skips, rank moves of 5+ places, and assets added or dropped. +- **TRAPI version**: each run records the TRAPI version it spoke (shown on the + run and in the runs list). A run whose version differs from its previous + run's says so on its page, in What changed, in the cycle board's flags and + in the Slack headline, since the change may be the upgrade, not the services. + Runs from before it was recorded (and imported ones) show none. +- **An asset**: its status per agent across the last 60 runs, the rank of the + expected answer and the number of results over time, and a table of runs. +- **Trends**: each agent's pass rate over full runs of a suite, overall and by + expected output. Skips are left out; runs with a target override or a single + query type aren't included. +- **Run grid**: every full acceptance run of a suite (no target override, no + single query type), one lane per environment, over the last 7–90 days. Each + card shows the run's number, date, pass rate, its change against the + previous run in that environment, and passed/failed/skipped counts; its color + says whether the pass rate went up, down, or held (within 0.1 pts), or that + it's the first run in the environment. Runs close together stack, and a line + joins each run to the next in its lane. Environments in + `RADIATOR_EXCLUDED_ENVS` (default `dev`) are greyed out and marked excluded. + Every run gets a number from the radiator (`#123`), in order of when it + started; imported runs are numbered oldest first. +- **Performance**: HelmsDeep's max sustainable concurrency per service over + time (per host, run type, profile, and environment), with checkpoint + verdicts. + +## In the Slack report + +Everything below goes to `SLACK_CHANNEL` once Zebrunner is gone. While +Zebrunner is configured (`ZE_*`), `SLACK_CHANNEL` stays Zebrunner's, with the +report it always got, and everything below goes to the radiator's own channel +(`RADIATOR_SLACK_WEBHOOK_URL`, `RADIATOR_SLACK_CHANNEL`), or isn't posted +without one (see the README's +[A Slack channel for the new radiator](../README.md#a-slack-channel-for-the-new-radiator)). + +When the radiator has the run, the harness's Slack report gains a headline +against the previous run of the same series, and a chart of the series up to +this run is posted under it: + +- **Acceptance runs:** pass rate and its change, regressions, fixes, and a link + to what changed; the chart is each agent's pass rate over the last 30 runs. +- **Performance runs:** for each service, its max sustainable concurrency and + its change since the service's previous run, and the checkpoint verdict; the + chart is that concurrency over the service's last 30 runs, with missed + checkpoints marked. A service's series is one host under one HelmsDeep run + type and profile, in one environment, since a `mixed` and a `default` run load + it differently. + +The charts use step lines: a run's value holds until the next run, so every rise +and drop shows plainly (the dashboard's trend charts do the same). The radiator +draws them (`history.png`, `performance.png`); the harness fetches them with its +token and uploads them, since Slack can't reach pages behind the login. Without +the radiator, or if it can't be reached, the report is what it always was. + +### Run cycles and the weekly digest + +`test-harness-cycle` runs acceptance, pathfinder and performance in one +environment and marks their runs with a shared `cycle_id`. When it finishes, it +posts the cycle's board (`/api/cycles/{id}/board.png`) and a message with each +test's headline, the regressions and fixes by agent, and the flags. The +digest, `test-harness-radiator digest`, posts `/api/digest.png` and the same +kind of message for every environment. See the README's +[Run cycles](../README.md#run-cycles-and-the-weekly-digest) for the command and +the flags. + +How the board is worked out (`radiator/cycles.py`): + +- **Which test a run is**: a run with performance results is performance; one + whose results are all pathfinder assets is pathfinder; anything else is + acceptance. +- **Compared with what**: each run against the previous finished run of its + series (same suite, environment, target override and query type), which is + last week's cycle in practice. Performance compares per service. +- **Verdict**: acceptance and pathfinder by regressions against fixes (agent + results that went from passing to not, or back): more regressions is + *Regressed*, more fixes *Improved*, an equal number *Mixed*, none *Steady*. + Performance by max sustainable concurrency: down 10% or more (or newly + missed checkpoints) is *Regressed*, up 10% or more *Improved*. +- **Flags**: thresholds are at the top of `cycles.py`, to tune as we learn + what's noise. +- **The digest** takes each environment's latest full run of each suite over + the week (no `--query_type`; acceptance and pathfinder without a target + override) and each performance target's latest run, whether or not a cycle + ran them. + +## API + +Uploads go under `/api/ingest`, with `Authorization: Bearer `: + +| Method | Path | Body | +|---|---|---| +| `POST` | `/api/ingest/runs` | `RunCreate` | +| `POST` | `/api/ingest/runs/{run_id}/results` | `ResultBatch` | +| `PATCH` | `/api/ingest/runs/{run_id}` | `RunFinish` | + +Every write is an upsert: runs on the client-generated `run_id`, results on +`(run_id, test_case_id, asset_id)`, and an asset's agent rows are replaced by +the newest upload. Retries and replays never duplicate. The asset ids come from +the Tests repo and are stable across runs, which is what history and trends key +on. + +Reading, with the token or a session: + +| Path | | +|---|---| +| `GET /api/runs?suite=&env=&limit=` | runs, newest first | +| `GET /api/runs/{run_id}` | a run with every result (`?results=false` for just the run) | +| `GET /api/runs/{run_id}/progress` | how far a run has got (assets tested of its total, counts, state); what its page polls | +| `GET /api/live` | the runs in progress, and a version that changes when runs start, upload or finish | +| `GET /api/runs/{run_id}/summary` | pass rate, previous pass rate, regressions, fixes | +| `GET /api/cycles?limit=20` | the latest run cycles | +| `GET /api/cycles/{cycle_id}` | a cycle's board: each test against its previous run, changes by agent, flags | +| `GET /api/cycles/{cycle_id}/board.png` | the board posted to Slack | +| `GET /api/digest?days=7&until=` | every environment's latest runs over the week | +| `GET /api/digest.png?days=7&until=` | the digest posted to Slack | +| `GET /api/runs/{run_id}/history.png?runs=30` | the pass-rate chart posted to Slack | +| `GET /api/runs/{run_id}/performance.png?runs=30` | the concurrency chart posted to Slack | + +## Deploying + +The Helm charts in [`deploy/helm`](../deploy/helm/README.md) do all of this: +`radiator` (the app, migrations, Service, Ingress, optionally Postgres and the +Zebrunner import) and `test-harness` (the run cycles, digest and sweep). Copy +each chart's `values.example.yaml` to a git-ignored `values..yaml`, fill +it in, and `helm upgrade --install`. Without Helm, the steps with the plain +manifests are: + +1. **Database.** Use an existing Postgres, or apply + `deploy/radiator-postgres.example.yaml`. Back it up: it's the only copy of + the history. +2. **Secrets.** Create `radiator-secrets` as described at the top of + `deploy/radiator.example.yaml`. +3. **The radiator.** Adapt and apply `deploy/radiator.example.yaml` (image tag, + hostname, TLS secret, ingress class). Migrations run in an init container on + every start; they're a no-op when the schema is current. +4. **The harness.** Add `RADIATOR_URL` and `RADIATOR_TOKEN` (the same + api-token) to `test-harness-secrets`; `deploy/cronjob.example.yaml` already + reads them. `RADIATOR_URL` can be the in-cluster service + (`http://radiator`); then also set `RADIATOR_PUBLIC_URL` to the address + people open, since the links in Slack are built from it. + The harness then reports to both Zebrunner and the radiator. +5. **History.** Run the Zebrunner import (below). +6. **Cutover**, after a couple of weeks of both: point people at the new UI, + then remove the Zebrunner `Reporter` and the `ZE_*` secrets. If the new + radiator had its own Slack channel meanwhile (`RADIATOR_SLACK_*`), point + `SLACK_*` at it and drop `RADIATOR_SLACK_*`. + +The image is built and pushed by the release workflow alongside the harness's. +To build it by hand, from the repo root: +`docker build -f radiator/Dockerfile -t testharness-radiator .` + +## Importing from Zebrunner + +``` +python -m radiator.zebrunner_import \ + --zebrunner-url "$ZE_BASE_URL" --refresh-token "$ZE_REFRESH_TOKEN" \ + --radiator-url https://radiator.example.org --radiator-token "$RADIATOR_TOKEN" \ + --since 2026-09-15 --dry-run --limit 5 +``` + +Drop `--dry-run`/`--limit` once that looks right; `deploy/radiator-import.example.yaml` +runs it in the cluster. It reads Zebrunner's API (launches, their tests' +labels, and each test's logs) and posts runs like the harness does. + +- **What comes across:** test case/asset ids, expected output, CURIEs, every + agent's status (from the labels the harness set), and from the logged report, + each agent's found/rank/score and the pks. `--no-logs` skips the logs: much + faster, but no ranks, scores or pks. +- **The environment** was never sent to Zebrunner, so it's worked out per + run: a pk only exists on the ARS that made it (and the harness asked the + ARS to retain each response), so the import asks each environment's ARS + (from the SmartAPI registry; `--ars ENV=URL` to override) for one of the + run's pks, and the one that has it is the run's environment. A run whose + pks no ARS has falls back to prod if it has prod-only agents, else no + environment. The log says which for each run, and the summary counts them. + `--env` records every run under one environment instead (no lookups). +- **What can't:** result counts, HTTP status and response times were never + recorded, so those show as "not recorded", never zero. +- **Skipped:** performance tests (pre-HelmsDeep, no summary). +- **Re-runnable:** imported runs get ids derived from the Zebrunner launch, so + a second run skips them, or updates them with `--force`. +- **Zebrunner versions:** its read API changed between versions. The import + speaks both: the v1 API (reporting-service 1.4x and later), which needs + `--project-id` (the `projectId` in the API calls Zebrunner's launches page + makes; your browser's network tab shows them), and the legacy API (eg + 1.28). By default it tries v1 and falls back; `--api v1|legacy` picks one. + On the legacy API it logs the fields of the first launch and test it + reads, so a dry run shows if anything is named differently. + +The v1 endpoints were read from Zebrunner's published images +(`zebrunner/reporting-service` 1.54.1, `zebrunner/test-execution-logs-service` +1.6.1); the legacy ones were checked against a 1.28 instance. Another version +may differ; `--dry-run --limit 5` shows quickly whether it works against yours. + +## Developing + +The quickest way to a running radiator is Docker Compose, from the repo root: +`docker compose up --build` (see the README's +[Trying it out with Docker Compose](../README.md#trying-it-out-with-docker-compose)). +To work on the radiator's code itself, without Docker: + +``` +# a Postgres to develop against, then: +export RADIATOR_DATABASE_URL=postgresql+psycopg://postgres@localhost:5432/radiator +export RADIATOR_API_TOKEN=dev-token RADIATOR_USERNAME=dev RADIATOR_PASSWORD=dev \ + RADIATOR_SESSION_SECRET=dev RADIATOR_SECURE_COOKIES=false +pip install -r radiator/requirements-test.txt +alembic -c radiator/alembic.ini upgrade head +python -m radiator.demo # made-up history to look at +uvicorn radiator.app:create_app --factory --reload +``` + +Tests need a database they may wipe: +`RADIATOR_TEST_DATABASE_URL=... pytest radiator/tests`. After changing +`models.py`, add a migration with +`alembic -c radiator/alembic.ini revision --autogenerate -m "..."`, review it, +and check `alembic -c radiator/alembic.ini check` is clean (CI runs it). + +To point a local harness run at it: `RADIATOR_URL=http://localhost:8000 +RADIATOR_TOKEN=dev-token test-harness load local_acceptance`. + +## When a run, or part of one, is missing from the dashboard + +Every run's log says where its results went and what arrived: + +- At the start: `Reporting to the Information Radiator at : `, + or `Not reporting to the Information Radiator (--local / RADIATOR_URL and + RADIATOR_TOKEN aren't set)`. +- At the end: `Information Radiator: has 612 of the 640 assets in + the suite`, then, if any are missing, a warning naming them + (`TestCase_12/Asset_3, ...`). Search the log for one of those ids: every + place an asset can be left out logs it with its test case and asset id + and why: a test case with no assets or objective, an expected output the + harness doesn't test, a query that couldn't be built (`couldn't build the + query for asset ...`), no answer from a service (`No answer from at + : ConnectError ...`, recorded with the harness's 418), or a record the + schema rejected (`couldn't record asset ... for the Information Radiator`). +- If an upload failed: `Upload to the Information Radiator failed (POST , + N asset results ...)` with the radiator's answer (eg the field a 422 + rejected) or the connection error. Nothing more is sent for that run; the + whole run is saved as `radiator_.json`, to replay with + `test-harness-radiator push`. + +The radiator's own log has a line for each run opened, each batch of results +and each run finished, and, for a rejected upload, which fields were wrong +(`Rejected upload POST ...: body.results.0.status: ...`). Set +`RADIATOR_LOG_LEVEL` (the chart's `config.logLevel`) to change how much it says. + +## Known gaps + +- ARA response times under the ARS aren't recorded: the harness only sees + ARS fan-out through 10s polling. The ARS's own message timestamps could + give them. +- HelmsDeep's `report.html` isn't uploaded yet (it still goes to Slack); the + radiator stores the `summary.json` it is drawn from. diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..5383816 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +# The harness's tests. The radiator's run separately, in their own +# environment and against Postgres: pytest radiator/tests +testpaths = tests +pythonpath = . diff --git a/radiator/Dockerfile b/radiator/Dockerfile new file mode 100644 index 0000000..01ad71f --- /dev/null +++ b/radiator/Dockerfile @@ -0,0 +1,30 @@ +# The Information Radiator. Build from the repo root, since it needs the +# shared radiator_schema package: +# docker build -f radiator/Dockerfile -t testharness-radiator . +FROM ghcr.io/translatorsri/renci-python-image:3.11.5 + +LABEL org.opencontainers.image.source /github.com/TranslatorSRI/TestHarness + +WORKDIR /app + +COPY radiator/requirements.txt radiator/requirements.txt +RUN pip install --no-cache-dir -r radiator/requirements.txt + +RUN MPLCONFIGDIR=/tmp/matplotlib python -c "import matplotlib.pyplot" && chmod -R a+rwX /tmp/matplotlib + +COPY radiator_schema radiator_schema +COPY radiator radiator +RUN rm -rf radiator/tests + +ENV PYTHONPATH=/app \ + PYTHONUNBUFFERED=1 \ + MPLCONFIGDIR=/tmp/matplotlib + +# the non-root user defined in the base image +USER nru + +EXPOSE 8000 + +# Migrations run separately (an init container in deploy/radiator.example.yaml): +# alembic -c radiator/alembic.ini upgrade head +CMD ["uvicorn", "radiator.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8000"] diff --git a/radiator/__init__.py b/radiator/__init__.py new file mode 100644 index 0000000..a38b76e --- /dev/null +++ b/radiator/__init__.py @@ -0,0 +1 @@ +"""The Information Radiator: an always-on dashboard of Test Harness results.""" diff --git a/radiator/alembic.ini b/radiator/alembic.ini new file mode 100644 index 0000000..3ad1bae --- /dev/null +++ b/radiator/alembic.ini @@ -0,0 +1,39 @@ +# Migrations for the Information Radiator. Run from the repo root: +# alembic -c radiator/alembic.ini upgrade head +# The database URL comes from RADIATOR_DATABASE_URL. +[alembic] +script_location = %(here)s/migrations +prepend_sys_path = %(here)s/.. +path_separator = os + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s diff --git a/radiator/api.py b/radiator/api.py new file mode 100644 index 0000000..104069f --- /dev/null +++ b/radiator/api.py @@ -0,0 +1,227 @@ +"""The JSON API: ingest for the harness, and a small read API for scripts.""" + +import logging +import uuid +from datetime import datetime + +from fastapi import APIRouter, Depends, HTTPException, Request, Response, status +from sqlalchemy.orm import Session + +import radiator_schema as schema +from radiator import board_png, cycles, history_png, ingest, queries +from radiator.auth import require_token, require_token_or_login +from radiator.models import Run + +logger = logging.getLogger("radiator") + +ingest_router = APIRouter( + prefix=schema.INGEST_PREFIX, dependencies=[Depends(require_token)] +) +read_router = APIRouter(prefix="/api", dependencies=[Depends(require_token_or_login)]) + + +def get_session(request: Request): + with request.app.state.sessionmaker() as session: + yield session + + +def _commit_or_404(session: Session, run_id: uuid.UUID, write): + try: + write() + except ingest.RunNotFound: + session.rollback() + logger.warning(f"Upload for unknown run {run_id}: it was never opened") + raise HTTPException( + status.HTTP_404_NOT_FOUND, + f"No run {run_id}: open it with POST {schema.INGEST_PREFIX}/runs first", + ) + session.commit() + + +@ingest_router.post("/runs", status_code=status.HTTP_201_CREATED) +def create_run(run: schema.RunCreate, session: Session = Depends(get_session)): + ingest.upsert_run(session, run) + session.commit() + logger.info( + f"Run {run.run_id} opened: {run.suite} in {run.env or 'no environment'}" + + (f", target {run.target}" if run.target else "") + + ( + f", {run.expected_assets} assets expected" + if run.expected_assets is not None + else "" + ) + ) + return {"run_id": str(run.run_id)} + + +@ingest_router.post("/runs/{run_id}/results") +def add_results( + run_id: uuid.UUID, + batch: schema.ResultBatch, + session: Session = Depends(get_session), +): + _commit_or_404( + session, run_id, lambda: ingest.upsert_results(session, run_id, batch) + ) + logger.info( + f"Run {run_id}: {len(batch.results)} asset results, " + f"{len(batch.performance)} performance results" + ) + return {"results": len(batch.results), "performance": len(batch.performance)} + + +@ingest_router.patch("/runs/{run_id}") +def finish_run( + run_id: uuid.UUID, + finish: schema.RunFinish, + session: Session = Depends(get_session), +): + _commit_or_404(session, run_id, lambda: ingest.finish_run(session, run_id, finish)) + logger.info(f"Run {run_id} finished") + return {"run_id": str(run_id)} + + +@read_router.get("/runs") +def list_runs( + suite: str | None = None, + env: str | None = None, + limit: int = 50, + session: Session = Depends(get_session), +): + runs = queries.list_runs(session, suite=suite, env=env, limit=min(limit, 500)) + return [queries.run_json(run) for run in runs] + + +@read_router.get("/runs/{run_id}") +def get_run( + run_id: uuid.UUID, results: bool = True, session: Session = Depends(get_session) +): + """A run, with every result unless ``results=false``.""" + if results: + run = queries.get_run(session, run_id) + else: + run = session.get(Run, run_id) + if run is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "No such run") + return queries.run_json(run, with_results=results) + + +def _run_or_404(session: Session, run_id: uuid.UUID) -> Run: + run = session.get(Run, run_id) + if run is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "No such run") + return run + + +@read_router.get("/runs/{run_id}/summary") +def get_run_summary(run_id: uuid.UUID, session: Session = Depends(get_session)): + """The run against the previous one in its series: pass rates, + regressions, fixes. For notifications such as the Slack report.""" + return queries.run_summary(session, _run_or_404(session, run_id)) + + +@read_router.get("/runs/{run_id}/progress") +def get_run_progress(run_id: uuid.UUID, session: Session = Depends(get_session)): + """How far a run has got: what the run page polls while it's running.""" + return queries.run_progress(session, _run_or_404(session, run_id)) + + +@read_router.get("/live") +def get_live(session: Session = Depends(get_session)): + """The runs in progress: what the runs list and the weekly page poll.""" + return queries.live_status(session) + + +@read_router.get("/runs/{run_id}/history.png") +def get_run_history_png( + run_id: uuid.UUID, runs: int = 30, session: Session = Depends(get_session) +): + """Pass rate by agent over the series' last ``runs`` runs, ending at this + one, as a PNG (for Slack, which can't show the dashboard's charts).""" + run = _run_or_404(session, run_id) + history = queries.series_history(session, run, limit=min(max(runs, 1), 120)) + return Response( + history_png.render(run, history), + media_type="image/png", + headers={"Cache-Control": "private, max-age=300"}, + ) + + +@read_router.get("/runs/{run_id}/performance.png") +def get_run_performance_png( + run_id: uuid.UUID, runs: int = 30, session: Session = Depends(get_session) +): + """Each of the run's services' max sustainable concurrency over its last + ``runs`` runs, ending at this one, as a PNG (for Slack).""" + run = queries.get_run(session, run_id) + if run is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "No such run") + if not run.performance: + raise HTTPException(status.HTTP_404_NOT_FOUND, "No performance results") + series = queries.performance_series(session, run, limit=min(max(runs, 1), 120)) + return Response( + history_png.render_performance(run, series), + media_type="image/png", + headers={"Cache-Control": "private, max-age=300"}, + ) + + +def _png(data: bytes) -> Response: + return Response( + data, media_type="image/png", headers={"Cache-Control": "private, max-age=300"} + ) + + +@read_router.get("/cycles") +def list_cycles(limit: int = 20, session: Session = Depends(get_session)): + """The latest run cycles, newest first.""" + return [ + {"cycle_id": str(cycle_id), "env": env, "started_at": started.isoformat()} + for cycle_id, env, started in cycles.recent_cycles( + session, limit=min(max(limit, 1), 200) + ) + ] + + +def _cycle_or_404(session: Session, cycle_id: uuid.UUID) -> cycles.Cycle: + cycle = cycles.get_cycle(session, cycle_id) + if cycle is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "No such cycle") + return cycle + + +@read_router.get("/cycles/{cycle_id}") +def get_cycle(cycle_id: uuid.UUID, session: Session = Depends(get_session)): + """A run cycle's board: each test against its previous run, with the + regressions and fixes by agent, and flags. For the Slack report.""" + return cycles.cycle_json(_cycle_or_404(session, cycle_id)) + + +@read_router.get("/cycles/{cycle_id}/board.png") +def get_cycle_board_png(cycle_id: uuid.UUID, session: Session = Depends(get_session)): + """The cycle's board as a PNG (for Slack).""" + return _png(board_png.board_png(_cycle_or_404(session, cycle_id))) + + +def _digest(session: Session, days: int, until: datetime | None) -> cycles.Digest: + return cycles.digest(session, until=until, days=min(max(days, 1), 31)) + + +@read_router.get("/digest") +def get_digest( + days: int = 7, + until: datetime | None = None, + session: Session = Depends(get_session), +): + """Every environment's latest runs of each test over the last ``days``.""" + return cycles.digest_json(_digest(session, days, until)) + + +@read_router.get("/digest.png") +def get_digest_png( + days: int = 7, + until: datetime | None = None, + session: Session = Depends(get_session), +): + """The weekly digest as a PNG (for Slack).""" + return _png(board_png.digest_png(_digest(session, days, until))) diff --git a/radiator/app.py b/radiator/app.py new file mode 100644 index 0000000..1b9f412 --- /dev/null +++ b/radiator/app.py @@ -0,0 +1,165 @@ +"""The Information Radiator web app. + +Run with ``uvicorn radiator.app:create_app --factory``. Behind the cluster +ingress, the login throttle reads each client's address from the +X-Forwarded-For entry the ingress adds (see ``auth.client_address``). +""" + +import logging +import os +from pathlib import Path +from typing import Optional +from urllib.parse import quote, urlparse + +from fastapi import FastAPI, Form, Request +from fastapi.exception_handlers import request_validation_exception_handler +from fastapi.exceptions import RequestValidationError +from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates +from sqlalchemy import text +from starlette.middleware.sessions import SessionMiddleware + +from radiator import api, web +from radiator.auth import ( + LoginThrottle, + check_credentials, + client_address, + is_logged_in, +) +from radiator.config import Settings +from radiator.db import make_sessionmaker + +HERE = Path(__file__).parent +logger = logging.getLogger("radiator") + + +def configure_logging() -> None: + """Send the radiator's own log lines (uploads, rejected uploads) to + stderr next to uvicorn's, at RADIATOR_LOG_LEVEL (default INFO).""" + if logger.handlers: + return + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s")) + logger.addHandler(handler) + logger.setLevel(os.getenv("RADIATOR_LOG_LEVEL", "INFO").upper()) + logger.propagate = False + + +SESSION_MAX_AGE_S = 30 * 24 * 3600 + + +def _safe_next(next_url: Optional[str]) -> str: + """Only redirect back to a path on this site after login.""" + if not next_url: + return "/" + parsed = urlparse(next_url) + if parsed.scheme or parsed.netloc or not next_url.startswith("/"): + return "/" + # browsers read "/\\host" like "//host", another site + if next_url[1:2] in ("/", "\\"): + return "/" + return next_url + + +def create_app(settings: Optional[Settings] = None) -> FastAPI: + configure_logging() + settings = settings or Settings.from_env() + app = FastAPI(title="Information Radiator", docs_url=None, redoc_url=None) + app.state.settings = settings + app.state.sessionmaker = make_sessionmaker(settings.database_url) + app.state.login_throttle = LoginThrottle() + templates = Jinja2Templates(directory=HERE / "templates") + web.register_template_helpers(templates) + app.state.templates = templates + + app.add_middleware( + SessionMiddleware, + secret_key=settings.session_secret, + session_cookie="radiator_session", + max_age=SESSION_MAX_AGE_S, + same_site="lax", + https_only=settings.secure_cookies, + ) + app.mount("/static", StaticFiles(directory=HERE / "static"), name="static") + app.include_router(api.ingest_router) + app.include_router(api.read_router) + app.include_router(web.router) + + @app.exception_handler(RequestValidationError) + async def rejected(request: Request, exc: RequestValidationError): + """An upload the schema rejects is lost to the dashboard: say exactly + what was wrong, here as well as in the harness's log.""" + if request.url.path.startswith(api.schema.INGEST_PREFIX): + errors = exc.errors() + details = "; ".join( + f"{'.'.join(str(p) for p in e.get('loc', ()))}: {e.get('msg')}" + for e in errors[:10] + ) + more = f" (and {len(errors) - 10} more)" if len(errors) > 10 else "" + logger.warning( + f"Rejected upload {request.method} {request.url.path}: " + f"{details}{more}" + ) + return await request_validation_exception_handler(request, exc) + + @app.exception_handler(web.LoginRequired) + async def login_required(request: Request, exc: web.LoginRequired): + target = request.url.path + if request.url.query: + target += f"?{request.url.query}" + return RedirectResponse( + f"/login?next={quote(target, safe='/')}", status_code=303 + ) + + @app.get("/healthz", include_in_schema=False) + def healthz(): + """Liveness/readiness: the app is up and can reach its database.""" + try: + with app.state.sessionmaker() as session: + session.execute(text("SELECT 1")) + except Exception: + return JSONResponse({"ok": False}, status_code=503) + return {"ok": True} + + @app.get("/login", response_class=HTMLResponse, include_in_schema=False) + def login_page(request: Request, next: Optional[str] = None): + if is_logged_in(request): + return RedirectResponse(_safe_next(next), status_code=303) + return templates.TemplateResponse( + request, "login.html", {"next": _safe_next(next), "error": None} + ) + + @app.post("/login", include_in_schema=False) + def login( + request: Request, + username: str = Form(...), + password: str = Form(...), + next: Optional[str] = Form(None), + ): + throttle: LoginThrottle = app.state.login_throttle + client = client_address(request) + error = None + if throttle.blocked(client): + error = "Too many failed attempts. Try again in a few minutes." + elif check_credentials(request, username, password): + throttle.succeeded(client) + request.session.clear() + request.session["user"] = settings.username + return RedirectResponse(_safe_next(next), status_code=303) + else: + throttle.failed(client) + error = "That username and password didn't match." + return templates.TemplateResponse( + request, + "login.html", + {"next": _safe_next(next), "error": error}, + status_code=401, + ) + + @app.post("/logout", include_in_schema=False) + def logout(request: Request): + request.session.clear() + return RedirectResponse("/login", status_code=303) + + return app diff --git a/radiator/auth.py b/radiator/auth.py new file mode 100644 index 0000000..d6ea4b3 --- /dev/null +++ b/radiator/auth.py @@ -0,0 +1,119 @@ +"""Authentication. + +Two ways in, and nothing is reachable without one of them (bar the health +check and the login page itself): + +* the UI: one shared username/password for the consortium, which sets a + signed session cookie; +* the API: a bearer token, for the harness to upload with and for scripts. + +The JSON read API accepts either, so the UI's own pages could use it too. +""" + +import hmac +import time +from collections import deque +from typing import Optional + +from fastapi import HTTPException, Request, status + + +def _equal(a: Optional[str], b: str) -> bool: + return a is not None and hmac.compare_digest(a.encode(), b.encode()) + + +def has_valid_token(request: Request) -> bool: + header = request.headers.get("authorization", "") + scheme, _, token = header.partition(" ") + return scheme.lower() == "bearer" and _equal( + token.strip(), request.app.state.settings.api_token + ) + + +def is_logged_in(request: Request) -> bool: + return request.session.get("user") == request.app.state.settings.username + + +def require_token(request: Request) -> None: + """Dependency for the ingest API: bearer token only.""" + if not has_valid_token(request): + raise HTTPException( + status.HTTP_401_UNAUTHORIZED, + "A valid bearer token is required", + headers={"WWW-Authenticate": "Bearer"}, + ) + + +def require_token_or_login(request: Request) -> None: + """Dependency for the read API.""" + if not (has_valid_token(request) or is_logged_in(request)): + raise HTTPException( + status.HTTP_401_UNAUTHORIZED, + "Log in, or send a bearer token", + headers={"WWW-Authenticate": "Bearer"}, + ) + + +def check_credentials(request: Request, username: str, password: str) -> bool: + settings = request.app.state.settings + # evaluate both, so a wrong username takes as long as a wrong password + user_ok = _equal(username, settings.username) + password_ok = _equal(password, settings.password) + return user_ok and password_ok + + +def client_address(request: Request) -> str: + """Who is logging in, for the throttle. + + Behind the ingress the peer is always the ingress, so the client comes + from X-Forwarded-For. Only its last entry is trusted: that's the one the + ingress itself adds, while anything before it is whatever the client sent. + """ + forwarded = request.headers.get("x-forwarded-for") + if forwarded: + last = forwarded.split(",")[-1].strip() + if last: + return last + return request.client.host if request.client else "unknown" + + +class LoginThrottle: + """Slow down password guessing against the one shared password. + + Counts failed logins per client address in a sliding window, in memory. + That is enough for the single replica this runs as; it resets on restart. + """ + + def __init__( + self, + max_failures: int = 10, + window_s: float = 15 * 60, + max_clients: int = 10_000, + ): + self.max_failures = max_failures + self.window_s = window_s + self.max_clients = max_clients + self._failures: dict[str, deque] = {} + + def _recent(self, key: str, now: float) -> deque: + failures = self._failures.get(key, deque()) + while failures and now - failures[0] > self.window_s: + failures.popleft() + return failures + + def blocked(self, key: str) -> bool: + return len(self._recent(key, time.monotonic())) >= self.max_failures + + def failed(self, key: str) -> None: + now = time.monotonic() + failures = self._recent(key, now) + failures.append(now) + self._failures[key] = failures + if len(self._failures) > self.max_clients: + # keep memory bounded: forget whoever has nothing recent + for other in list(self._failures): + if not self._recent(other, now): + del self._failures[other] + + def succeeded(self, key: str) -> None: + self._failures.pop(key, None) diff --git a/radiator/board_png.py b/radiator/board_png.py new file mode 100644 index 0000000..59ed903 --- /dev/null +++ b/radiator/board_png.py @@ -0,0 +1,328 @@ +"""The cycle board and the weekly digest, drawn as PNGs for Slack. + +Both are tables first: a row per test (the board) or per environment (the +digest), with the headline number, its change against the previous run, and a +verdict that always comes as a word beside its color. The board adds a small +step line of each test's last runs, so a one-off dip reads differently from a +slide. +""" + +import io +from datetime import datetime, timezone +from typing import Optional + +from radiator import cycles +from radiator.cycles import Cycle, Digest, Row + +# Light-mode values of the page's tokens (static/style.css). +INK, INK_2, MUTED = "#0b0b0b", "#52514e", "#6f6d68" +SURFACE, BAND, RULE = "#fcfcfb", "#f3f2ef", "#e1e0d9" +# Status colors, reserved for verdicts and never used for anything else. +VERDICT_COLORS = { + cycles.REGRESSED: "#d03b3b", + cycles.MIXED: "#eda100", + cycles.IMPROVED: "#0ca30c", + cycles.STEADY: "#898781", + cycles.FIRST: "#2a78d6", + cycles.NO_DATA: "#52514e", +} +FLAG = "⚑" + + +class _Canvas: + """A figure in px, y down, with text always above the shapes.""" + + def __init__(self, width: float, height: float): + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + plt.rcParams["font.family"] = "DejaVu Sans" + self.plt = plt + self.width, self.height = width, height + self.fig = plt.figure(figsize=(width / 100, height / 100), dpi=100) + self.fig.patch.set_facecolor(SURFACE) + self.ax = self.fig.add_axes([0, 0, 1, 1]) + self.ax.set_xlim(0, width) + self.ax.set_ylim(height, 0) + self.ax.axis("off") + + def text(self, x, y, s, size, color=INK, weight="normal", ha="left", va="baseline"): + return self.ax.text( + x, + y, + s, + fontsize=size * 0.75, + color=color, + weight=weight, + ha=ha, + va=va, + zorder=6, + ) + + def width_of(self, artist) -> float: + return artist.get_window_extent(renderer=self.fig.canvas.get_renderer()).width + + def rule(self, y, x1=32, x2=None): + self.ax.plot([x1, x2 or self.width - 32], [y, y], color=RULE, lw=1, zorder=1) + + def band(self, y, h): + from matplotlib.patches import Rectangle + + self.ax.add_patch( + Rectangle((0, y), self.width, h, facecolor=BAND, edgecolor="none", zorder=0) + ) + + def dot(self, x, y, r, color, ring=SURFACE, lw=2.0): + from matplotlib.patches import Circle + + self.ax.add_patch( + Circle((x, y), r, facecolor=color, edgecolor=ring, lw=lw, zorder=5) + ) + + def verdict(self, x, y, verdict, size=14): + """A verdict: its color, then its word.""" + self.dot(x + 7, y - size * 0.32, 7, VERDICT_COLORS[verdict], lw=0) + return self.text(x + 20, y, cycles.VERDICT_LABELS[verdict], size, INK, "bold") + + def legend(self, y, verdicts): + x = self.width - 32 + for verdict in reversed(verdicts): + t = self.text(x, y, cycles.VERDICT_LABELS[verdict], 13, INK_2, ha="right") + w = self.width_of(t) + self.dot(x - w - 12, y - 4.5, 6, VERDICT_COLORS[verdict], lw=0) + x -= w + 42 + + def png(self) -> bytes: + buf = io.BytesIO() + self.fig.savefig(buf, format="png", dpi=100, facecolor=SURFACE) + self.plt.close(self.fig) + return buf.getvalue() + + +def _day(when: datetime) -> str: + return when.astimezone(timezone.utc).strftime("%a %b %-d") + + +def _sparkline(c: _Canvas, x0, x1, y0, y1, values: list[Optional[float]], pct: bool): + """A step line of the last runs, oldest first; this run is the dark dot.""" + points = [(i, v) for i, v in enumerate(values) if v is not None] + if not points: + c.text( + (x0 + x1) / 2, (y0 + y1) / 2, "no data", 12, MUTED, ha="center", va="center" + ) + return + lo, hi = min(v for _, v in points), max(v for _, v in points) + # keep small wobbles small: at least 10 pts (or 20% of the value) of range + floor = 0.10 if pct else max(hi * 0.2, 1.0) + if hi - lo < floor: + mid = (hi + lo) / 2 + lo, hi = mid - floor / 2, mid + floor / 2 + n = max(len(values), 2) + + def px(i): + return x0 + i * (x1 - x0) / (n - 1) + + def py(v): + return y1 - (v - lo) / (hi - lo) * (y1 - y0) + + for (i1, v1), (i2, v2) in zip(points, points[1:]): + c.ax.plot( + [px(i1), px(i2), px(i2)], + [py(v1), py(v1), py(v2)], + color=INK_2, + lw=1.6, + zorder=3, + solid_joinstyle="miter", + ) + for i, v in points[:-1]: + c.dot(px(i), py(v), 3.2, INK_2, lw=1.5) + last_i, last_v = points[-1] + c.dot(px(last_i), py(last_v), 5.5, INK, lw=2) + + +BOARD_W = 1400 +BOARD_COLS = { + "test": 32, + "result": 330, + "change": 520, + "spark": 790, + "verdict": 1095, + "flags": 1260, +} +BOARD_ROW = 78 +BOARD_TOP = 150 + + +def board_png(cycle: Cycle) -> bytes: + rows = cycle.rows + env = cycle.env or "mixed environments" + height = BOARD_TOP + BOARD_ROW * max(len(rows), 1) + 64 + c = _Canvas(BOARD_W, height) + c.text( + 32, 44, f"{env.upper()} run cycle · {_day(cycle.started_at)}", 24, weight="bold" + ) + took = "" + if cycle.ended_at is not None: + minutes = (cycle.ended_at - cycle.started_at).total_seconds() / 60 + took = f" · took {int(minutes // 60)} h {int(minutes % 60):02d} min" + c.text( + 32, + 72, + f"Each test against its previous run in {env}{took}", + 14, + INK_2, + ) + c.legend(44, [cycles.REGRESSED, cycles.MIXED, cycles.IMPROVED, cycles.STEADY]) + + cols = BOARD_COLS + head_y = BOARD_TOP - 16 + for key, title in [ + ("test", "Test"), + ("result", "Result"), + ("change", "Change"), + ("spark", f"Last {cycles.HISTORY_RUNS} runs"), + ("verdict", "Verdict"), + ("flags", "Flags"), + ]: + c.text(cols[key], head_y, title, 12, MUTED) + c.rule(BOARD_TOP - 6) + if not rows: + c.text(32, BOARD_TOP + 40, "No results in this cycle yet.", 15, INK_2) + for i, row in enumerate(rows): + y0 = BOARD_TOP + i * BOARD_ROW + cy = y0 + BOARD_ROW / 2 + c.text(cols["test"], cy - 4, row.label, 16, weight="bold") + c.text(cols["test"], cy + 18, f"{row.run.suite} · #{row.run.number}", 12, MUTED) + c.text(cols["result"], cy + 2, row.value_text, 24, weight="bold") + c.text(cols["result"], cy + 22, row.unit, 12, MUTED) + c.text(cols["change"], cy - 4, row.change_text, 15, weight="bold") + c.text(cols["change"], cy + 18, row.detail_text, 12, INK_2) + _sparkline( + c, + cols["spark"], + cols["spark"] + 250, + y0 + 16, + y0 + BOARD_ROW - 16, + row.history, + pct=row.perf is None, + ) + c.verdict(cols["verdict"], cy + 5, row.verdict) + if row.flags: + c.text(cols["flags"], cy + 5, f"{FLAG} {len(row.flags)}", 15, INK, "bold") + else: + c.text(cols["flags"], cy + 5, "–", 15, MUTED) + c.rule(y0 + BOARD_ROW) + c.text( + 32, + height - 22, + "Regressions and fixes count agent results that went from passing to not " + "passing, or back. Flags are listed in the message.", + 12, + MUTED, + ) + return c.png() + + +DIGEST_W = 1500 +DIGEST_LABEL_W = 150 +DIGEST_FLAGS_W = 110 +DIGEST_ENTRY = 50 +DIGEST_TOP = 150 + + +def _entry(c: _Canvas, x, y, row: Row, perf_name: bool): + """One test in a digest cell: verdict, number, change; then detail.""" + c.dot(x + 7, y - 6, 7, VERDICT_COLORS[row.verdict], lw=0) + head = ( + f"{row.perf.helmsdeep_target or row.perf.host} " + if perf_name and row.perf + else "" + ) + t = c.text(x + 20, y, f"{head}{row.value_text}", 17, weight="bold") + c.text( + x + 28 + c.width_of(t), + y, + f"{row.change_text} · {cycles.VERDICT_LABELS[row.verdict]}", + 13, + INK_2, + ) + detail = row.detail_text if row.perf is None else row.detail_text + c.text( + x + 20, + y + 19, + f"{detail} · #{row.run.number}, {_day(row.run.started_at)}", + 11.5, + MUTED, + ) + + +def digest_png(d: Digest) -> bytes: + kinds = cycles.KINDS + heights = [] + for env in d.envs: + entries = max([len(env.cell(k)) for k in kinds] + [1]) + heights.append(entries * DIGEST_ENTRY + 30) + height = DIGEST_TOP + sum(heights) + 64 + c = _Canvas(DIGEST_W, height) + c.text( + 32, + 44, + f"Weekly test status · {d.since:%b %-d} – {d.until:%b %-d}", + 24, + weight="bold", + ) + c.text( + 32, + 72, + "Each environment's latest runs this week, against the run before", + 14, + INK_2, + ) + c.legend(44, [cycles.REGRESSED, cycles.MIXED, cycles.IMPROVED, cycles.STEADY]) + col_w = (DIGEST_W - 32 - DIGEST_LABEL_W - DIGEST_FLAGS_W) / len(kinds) + xs = [DIGEST_LABEL_W + i * col_w for i in range(len(kinds))] + for x, kind in zip(xs, kinds): + c.text(x, DIGEST_TOP - 16, cycles.KIND_LABELS[kind], 12, MUTED) + c.text(DIGEST_W - 32 - DIGEST_FLAGS_W + 12, DIGEST_TOP - 16, "Flags", 12, MUTED) + c.rule(DIGEST_TOP - 6) + if not d.envs: + c.text(32, DIGEST_TOP + 40, "No runs this week.", 15, INK_2) + y0 = DIGEST_TOP + for i, (env, h) in enumerate(zip(d.envs, heights)): + if i % 2: + c.band(y0, h) + c.text(32, y0 + 34, (env.env or "no environment").upper(), 16, weight="bold") + for x, kind in zip(xs, kinds): + rows = env.cell(kind) + if not rows: + c.text(x, y0 + 34, "not run this week", 13, MUTED) + for j, row in enumerate(rows): + _entry( + c, + x, + y0 + 34 + j * DIGEST_ENTRY, + row, + perf_name=kind == cycles.PERFORMANCE, + ) + n_flags = len(env.flags) + c.text( + DIGEST_W - 32 - DIGEST_FLAGS_W + 12, + y0 + 34, + f"{FLAG} {n_flags}" if n_flags else "–", + 15, + INK if n_flags else MUTED, + "bold" if n_flags else "normal", + ) + y0 += h + c.rule(y0) + c.text( + 32, + height - 22, + "Full runs only. Regressions and fixes count agent results that changed " + "between passing and not passing. Flags are listed in the message.", + 12, + MUTED, + ) + return c.png() diff --git a/radiator/charts.py b/radiator/charts.py new file mode 100644 index 0000000..99f3fb6 --- /dev/null +++ b/radiator/charts.py @@ -0,0 +1,229 @@ +"""Server-rendered SVG charts. + +Charts are drawn here rather than by a JS library, so pages work without a +frontend build and render the same for everyone. ``static/charts.js`` adds the +hover layer (a crosshair that snaps to the nearest run and a tooltip listing +every series there) from the data embedded in each chart. + +Specs follow the house rules: 2px step lines (each run's value holds until +the next run), >=8px end dots with a 2px surface ring, hairline solid +gridlines, one y-axis, a legend for 2+ series, and every chart paired with a +table view in its template. +""" + +import json +import math +from dataclasses import dataclass, field +from datetime import datetime +from typing import Callable, Optional + +from markupsafe import Markup, escape + +AGENT_COLOR_SLOTS = { + "ars": 1, + "aragorn": 2, + "shepherd-aragorn": 2, + "arax": 3, + "shepherd-arax": 3, + "biothings-explorer": 4, + "shepherd-bte": 4, + "bte": 4, + "improving-agent": 5, + "unsecret-agent": 6, + "cqs": 7, +} + + +# Approximate on-screen widths of the page layouts, so text renders near 1:1. +WIDTH_FULL = 1180 +WIDTH_HALF = 570 +WIDTH_THIRD = 360 + + +def agent_color(agent: str) -> str: + """The CSS color for an agent. Follows the agent, never its position.""" + return f"var(--agent-{AGENT_COLOR_SLOTS.get(agent, 8)})" + + +@dataclass +class Series: + name: str + color: str + # (x, y) with y None for a gap; x is when the run started + points: list[tuple[datetime, Optional[float]]] = field(default_factory=list) + + +def nice_ticks(lo: float, hi: float, count: int = 4) -> list[float]: + if hi == lo: + hi = lo + 1 + raw = (hi - lo) / count + magnitude = 10 ** math.floor(math.log10(raw)) + step = next(m * magnitude for m in (1, 2, 2.5, 5, 10) if m * magnitude >= raw) + start = math.floor(lo / step) * step + ticks = [] + value = start + while value <= hi + step * 1e-9: + ticks.append(round(value, 10)) + value += step + if ticks[-1] < hi: + ticks.append(round(value, 10)) + return ticks + + +def _date_ticks(x0: float, x1: float, count: int = 5) -> list[float]: + if x1 == x0: + return [x0] + return [x0 + (x1 - x0) * i / (count - 1) for i in range(count)] + + +def line_chart( + series: list[Series], + *, + y_format: Callable[[float], str] = lambda v: f"{v:g}", + y_min: Optional[float] = None, + y_max: Optional[float] = None, + invert: bool = False, + width: int = 720, + height: int = 220, + label: str = "", +) -> Markup: + """A time-series line chart, one line per series, on a single y-axis. + + ``invert`` puts small values at the top, for ranks (1 is best). + + The SVG scales to its container, text included, so ``width`` should be + about the width it is shown at; the ``WIDTH_*`` constants match the + layouts the pages use. + """ + values = [y for s in series for _, y in s.points if y is not None] + xs = sorted({x for s in series for x, _ in s.points}) + if not values or not xs: + return Markup('

No data for this range.

') + + lo = min(values) if y_min is None else y_min + hi = max(values) if y_max is None else y_max + ticks = nice_ticks(lo, hi) + if y_min is not None and ticks[0] < y_min: + # eg ranks start at 1: label the floor itself, not the 0 below it + ticks[0] = y_min + lo, hi = min(ticks[0], lo), max(ticks[-1], hi) + + left, right, top, bottom = 48, 16, 10, 26 + plot_w, plot_h = width - left - right, height - top - bottom + t0, t1 = xs[0].timestamp(), xs[-1].timestamp() + + def sx(x: datetime) -> float: + if t1 == t0: + return left + plot_w / 2 + return left + (x.timestamp() - t0) / (t1 - t0) * plot_w + + def sy(y: float) -> float: + frac = (y - lo) / (hi - lo) if hi != lo else 0.5 + if invert: + frac = 1 - frac + return top + plot_h - frac * plot_h + + parts = [ + f'' + ] + for tick in ticks: + y = sy(tick) + parts.append( + f'' + f'{escape(y_format(tick))}' + ) + base_y = top + plot_h + parts.append( + f'' + ) + for i, t in enumerate(_date_ticks(t0, t1, 5 if t1 > t0 else 1)): + x = left + (t - t0) / (t1 - t0) * plot_w if t1 > t0 else left + plot_w / 2 + anchor = ( + "start" + if i == 0 and t1 > t0 + else ("end" if x >= width - right - 1 else "middle") + ) + when = datetime.fromtimestamp(t, tz=xs[0].tzinfo).strftime("%b %-d") + parts.append( + f'{when}' + ) + + for s in series: + runs: list[list[tuple[float, float]]] = [[]] + for x, y in sorted(s.points, key=lambda p: p[0]): + if y is None: + runs.append([]) + else: + runs[-1].append((sx(x), sy(y))) + for run in runs: + if len(run) > 1: + # steps: a run's value holds until the next run, so every + # change shows as a clean rise or drop + stepped = [run[0]] + for px, py in run[1:]: + stepped += [(px, stepped[-1][1]), (px, py)] + d = " ".join(f"{px:.1f},{py:.1f}" for px, py in stepped) + parts.append( + f'' + ) + elif len(run) == 1: + px, py = run[0] + parts.append( + f'' + ) + last = next( + ( + (x, y) + for x, y in sorted(s.points, key=lambda p: p[0], reverse=True) + if y is not None + ), + None, + ) + if last is not None: + parts.append( + f'' + ) + + parts.append( + f'' + ) + parts.append( + f'' + ) + parts.append("") + + # what the hover layer needs: x positions (in viewBox units) and, at each, + # every series' formatted value + hover = { + "width": width, + "xs": [round(sx(x), 1) for x in xs], + "titles": [x.strftime("%b %-d %Y, %H:%M") for x in xs], + "series": [ + { + "name": s.name, + "color": s.color, + "values": [ + (lambda y: None if y is None else y_format(y))( + dict(s.points).get(x) + ) + for x in xs + ], + } + for s in series + ], + } + legend = "" + if len(series) > 1: + legend = ( + '
' + + "".join( + f'{escape(s.name)}' + for s in series + ) + + "
" + ) + data = escape(json.dumps(hover)) + return Markup( + f'
{"".join(parts)}
{legend}' + ) diff --git a/radiator/config.py b/radiator/config.py new file mode 100644 index 0000000..10e89b7 --- /dev/null +++ b/radiator/config.py @@ -0,0 +1,48 @@ +"""Settings, all from the environment so they can come from a k8s secret.""" + +import os +from dataclasses import dataclass + + +class ConfigError(RuntimeError): + """A required setting is missing.""" + + +@dataclass(frozen=True) +class Settings: + database_url: str + # Bearer token for the API, which the harness uses to upload. + api_token: str + # The one shared login for the consortium. + username: str + password: str + # Signs the session cookie. Changing it logs everyone out. + session_secret: str + # Only send the session cookie over HTTPS. Off just for local development. + secure_cookies: bool = True + # Environments the run grid greys out and gives no trend: their runs are + # shown, but don't count. Comma-separated in RADIATOR_EXCLUDED_ENVS. + excluded_envs: tuple[str, ...] = ("dev",) + + @classmethod + def from_env(cls) -> "Settings": + def required(name): + value = os.getenv(name) + if not value: + raise ConfigError(f"{name} must be set") + return value + + return cls( + database_url=required("RADIATOR_DATABASE_URL"), + api_token=required("RADIATOR_API_TOKEN"), + username=required("RADIATOR_USERNAME"), + password=required("RADIATOR_PASSWORD"), + session_secret=required("RADIATOR_SESSION_SECRET"), + secure_cookies=os.getenv("RADIATOR_SECURE_COOKIES", "true").lower() + not in ("0", "false", "no"), + excluded_envs=tuple( + env.strip().lower() + for env in os.getenv("RADIATOR_EXCLUDED_ENVS", "dev").split(",") + if env.strip() + ), + ) diff --git a/radiator/cycles.py b/radiator/cycles.py new file mode 100644 index 0000000..6a4d939 --- /dev/null +++ b/radiator/cycles.py @@ -0,0 +1,741 @@ +"""Run cycles, the board, flags, and the weekly digest. + +A run cycle is acceptance, then pathfinder, then performance, run back to back +in one environment by ``test-harness-cycle``; its runs share a ``cycle_id``. +The *board* is one row per test in the cycle, each against the previous run of +the same test in that environment: the headline number, its change, and a +verdict (regressed, improved, mixed, steady). *Flags* point out oddities a pass +rate hides: an agent that stopped returning results, started erroring, got much +slower, or ranked the expected answer much lower while still passing. + +The *digest* is the same rows for every environment: each one's latest runs of +each test over the past week, whether or not a cycle ran them. +""" + +import statistics +import uuid +from collections import defaultdict +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from typing import Iterable, Optional + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from radiator import queries +from radiator.grid import ENV_ORDER +from radiator.models import AssetResult, PerformanceResult, Run + +ACCEPTANCE, PATHFINDER, PERFORMANCE = "acceptance", "pathfinder", "performance" +KINDS = [ACCEPTANCE, PATHFINDER, PERFORMANCE] +KIND_LABELS = { + ACCEPTANCE: "Acceptance", + PATHFINDER: "Pathfinder", + PERFORMANCE: "Performance", +} + +REGRESSED, MIXED, IMPROVED, STEADY, FIRST, NO_DATA = ( + "regressed", + "mixed", + "improved", + "steady", + "first", + "nodata", +) +VERDICT_LABELS = { + REGRESSED: "Regressed", + MIXED: "Mixed", + IMPROVED: "Improved", + STEADY: "Steady", + FIRST: "First run", + NO_DATA: "No result", +} + +# A concurrency change smaller than this is noise between runs. +PERF_STEADY_BAND = 0.10 + +# Flag thresholds. Kept together so they're easy to tune. +FLAG_MIN_ASSETS = 3 # no results / errors: on at least this many assets +SLOWER_RATIO = 2.0 # median response time at least doubled... +SLOWER_MIN_S = 5.0 # ...and went up by at least this many seconds +RANK_DROP = 30 # still passing, but the expected answer fell this many places +CONCURRENCY_DROP = 0.25 # max sustainable concurrency fell by this fraction + +# How many assets a flag or a change list names before "and N more". +NAMED_ASSETS = 5 + +HISTORY_RUNS = 8 + + +@dataclass +class AssetRef: + test_case_id: str + asset_id: str + name: Optional[str] + + @classmethod + def of(cls, asset: AssetResult) -> "AssetRef": + return cls(asset.test_case_id, asset.asset_id, asset.name) + + @property + def label(self) -> str: + return self.name or f"{self.test_case_id}/{self.asset_id}" + + +@dataclass +class Flag: + kind: str + who: str # an agent, or a performance target + text: str + run_id: uuid.UUID + assets: list[AssetRef] = field(default_factory=list) + + +@dataclass +class AgentChanges: + agent: str + regressions: list[AssetRef] = field(default_factory=list) + fixed: list[AssetRef] = field(default_factory=list) + + +@dataclass +class Row: + run: Run + kind: str + verdict: str + value: Optional[float] # pass rate (0..1), or max sustainable concurrency + previous: Optional[float] + previous_run: Optional[Run] + history: list[Optional[float]] # oldest first, ending with this run + regressions: Optional[int] = None + fixed: Optional[int] = None + counts: dict = field(default_factory=dict) + perf: Optional[PerformanceResult] = None + flags: list[Flag] = field(default_factory=list) + changes: list[AgentChanges] = field(default_factory=list) + + @property + def label(self) -> str: + if self.perf is not None: + return f"{KIND_LABELS[self.kind]} · {_perf_name(self.perf)}" + return KIND_LABELS[self.kind] + + @property + def value_text(self) -> str: + if self.value is None: + if self.perf is not None: + return "no result" + return "all skipped" + if self.perf is not None: + return f"{self.value:.1f}" + return f"{self.value * 100:.1f}%" + + @property + def unit(self) -> str: + return "max concurrency" if self.perf is not None else "pass rate" + + @property + def change_text(self) -> str: + if self.previous_run is None: + return "first run" + if self.value is None or self.previous is None: + return "–" + delta = self.value - self.previous + if self.perf is not None: + if abs(delta) < 0.05: + return "no change" + pct = f" ({delta / self.previous:+.0%})" if self.previous else "" + return f"{_arrow(delta)}{abs(delta):.1f}{pct}" + if abs(delta * 100) < 0.1: + return "no change" + return f"{_arrow(delta)}{abs(delta) * 100:.1f} pts" + + @property + def detail_text(self) -> str: + if self.perf is not None: + if self.perf.error: + return f"run failed: {self.perf.error}" + if self.perf.checkpoints_passed is True: + return "checkpoints passed" + if self.perf.checkpoints_passed is False: + return "checkpoints missed" + return "no checkpoints" + if self.regressions is None: + passed = self.counts.get("PASSED", 0) + total = sum(self.counts.values()) + return f"{passed} of {total} passed" + return f"{_plural(self.regressions, 'regression')} · {self.fixed} fixed" + + +def _arrow(delta: float) -> str: + return "▲" if delta > 0 else "▼" + + +def _plural(n: int, word: str) -> str: + return f"{n} {word}" + ("" if n == 1 else "s") + + +def _perf_name(perf: PerformanceResult) -> str: + return perf.helmsdeep_target or perf.host + + +# --- which test a run is --------------------------------------------------------- + + +def run_kinds(session: Session, run_ids: list[uuid.UUID]) -> dict[uuid.UUID, str]: + """acceptance, pathfinder or performance, per run, from what it recorded. + + A run with performance results is a performance run; one whose assets are + all pathfinder assets is a pathfinder run; anything else is acceptance. + """ + kinds: dict = {run_id: ACCEPTANCE for run_id in run_ids} + if not run_ids: + return kinds + asset_kinds: dict = defaultdict(set) + for run_id, kind in session.execute( + select(AssetResult.run_id, AssetResult.kind) + .where(AssetResult.run_id.in_(run_ids)) + .distinct() + ): + asset_kinds[run_id].add(kind) + for run_id, kinds_seen in asset_kinds.items(): + if kinds_seen == {PATHFINDER}: + kinds[run_id] = PATHFINDER + for run_id in session.scalars( + select(PerformanceResult.run_id) + .where(PerformanceResult.run_id.in_(run_ids)) + .distinct() + ): + kinds[run_id] = PERFORMANCE + return kinds + + +# --- flags ------------------------------------------------------------------------ + + +def _had_results(agent) -> bool: + if agent.n_results is not None: + return agent.n_results > 0 + return agent.status in ("PASSED", "FAILED") + + +def _no_results(agent) -> bool: + return agent.status == "NO_RESULTS" or agent.n_results == 0 + + +def _errored(agent) -> bool: + return agent.status == "ERROR" or (agent.http_status or 0) >= 400 + + +def _named(assets: list[AssetRef]) -> str: + names = ", ".join(a.label for a in assets[:NAMED_ASSETS]) + more = len(assets) - NAMED_ASSETS + return names + (f" and {more} more" if more > 0 else "") + + +def result_flags(base: Optional[Run], run: Run) -> list[Flag]: + """Oddities in an acceptance or pathfinder run against the previous run.""" + if base is None: + return [] + before_assets = {(a.test_case_id, a.asset_id): a for a in base.assets} + lost: dict = defaultdict(list) + errors: dict = defaultdict(list) + dropped: dict = defaultdict(list) + times_before: dict = defaultdict(list) + times_after: dict = defaultdict(list) + for asset in run.assets: + previous = before_assets.get((asset.test_case_id, asset.asset_id)) + if previous is None: + continue + before_agents = {a.agent: a for a in previous.agents} + for after in asset.agents: + before = before_agents.get(after.agent) + if before is None: + continue + ref = AssetRef.of(asset) + if _had_results(before) and _no_results(after): + lost[after.agent].append(ref) + if _errored(after) and not _errored(before): + errors[after.agent].append(ref) + if ( + before.status == after.status == "PASSED" + and before.rank is not None + and after.rank is not None + and after.rank - before.rank >= RANK_DROP + ): + dropped[after.agent].append( + (after.rank - before.rank, ref, before, after) + ) + if before.response_time_s is not None and after.response_time_s is not None: + times_before[after.agent].append(before.response_time_s) + times_after[after.agent].append(after.response_time_s) + + flags = [] + for agent, refs in lost.items(): + if len(refs) >= FLAG_MIN_ASSETS: + flags.append( + Flag( + "no_results", + agent, + f"{agent}: no results on {len(refs)} assets that had results " + f"last run ({_named(refs)})", + run.id, + refs, + ) + ) + for agent, refs in errors.items(): + if len(refs) >= FLAG_MIN_ASSETS: + flags.append( + Flag( + "errors", + agent, + f"{agent}: errors on {len(refs)} assets that didn't error " + f"last run ({_named(refs)})", + run.id, + refs, + ) + ) + for agent in times_after: + before_s = statistics.median(times_before[agent]) + after_s = statistics.median(times_after[agent]) + if after_s >= SLOWER_RATIO * before_s and after_s - before_s >= SLOWER_MIN_S: + flags.append( + Flag( + "slower", + agent, + f"{agent}: median response time {before_s:.1f}s → {after_s:.1f}s", + run.id, + ) + ) + for agent, drops in dropped.items(): + drops.sort(key=lambda d: -d[0]) + refs = [ref for _, ref, _, _ in drops] + worst = ", ".join( + f"{ref.label} ({before.rank} → {after.rank})" + for _, ref, before, after in drops[:NAMED_ASSETS] + ) + more = len(drops) - NAMED_ASSETS + flags.append( + Flag( + "rank_drop", + agent, + f"{agent}: expected answer down {RANK_DROP}+ places on " + f"{_plural(len(drops), 'asset')}, still passing: {worst}" + + (f" and {more} more" if more > 0 else ""), + run.id, + refs, + ) + ) + flags.sort(key=lambda f: (queries.agent_sort_key(f.who), f.kind)) + return flags + + +def performance_flags( + run: Run, perf: PerformanceResult, previous: Optional[PerformanceResult] +) -> list[Flag]: + """Oddities in a performance result against the previous one.""" + if previous is None: + return [] + name = _perf_name(perf) + was = previous.max_sustainable_concurrency + now = perf.max_sustainable_concurrency + flags = [] + if was is not None and now is None: + reason = f"run failed: {perf.error}" if perf.error else "no stage met the SLO" + flags.append( + Flag("perf_failed", name, f"{name}: {reason} (was {was:.1f})", run.id) + ) + elif was and now is not None and now <= (1 - CONCURRENCY_DROP) * was: + flags.append( + Flag( + "concurrency_drop", + name, + f"{name}: max sustainable concurrency {was:.1f} → {now:.1f} " + f"({(now - was) / was:+.0%})", + run.id, + ) + ) + if previous.checkpoints_passed is True and perf.checkpoints_passed is False: + flags.append( + Flag( + "checkpoints", + name, + f"{name}: missed checkpoints it passed last run", + run.id, + ) + ) + return flags + + +# --- rows --------------------------------------------------------------------------- + + +def trapi_flag(previous: Optional[Run], run: Run) -> list[Flag]: + """A flag when the TRAPI version changed since the previous run, so its + changes can be read as the upgrade's rather than as regressions.""" + change = queries.trapi_change(previous, run) + if change is None: + return [] + before, after = change + return [ + Flag( + "trapi_change", + "TRAPI", + f"TRAPI {before} → {after} since the previous run: changes here may " + "come from the version change", + run.id, + ) + ] + + +def _changes_by_agent(diff: queries.Diff) -> list[AgentChanges]: + by_agent: dict = {} + for kind in ("regression", "fixed"): + for change in diff.changes.get(kind, []): + entry = by_agent.setdefault(change.agent, AgentChanges(change.agent)) + target = entry.regressions if kind == "regression" else entry.fixed + target.append(AssetRef.of(change.asset)) + return [by_agent[a] for a in sorted(by_agent, key=queries.agent_sort_key)] + + +def _result_verdict(row: Row) -> str: + if row.value is None: + return NO_DATA + if row.previous_run is None: + return FIRST + # by the net change; both counts are always shown alongside + if row.regressions > row.fixed: + return REGRESSED + if row.fixed > row.regressions: + return IMPROVED + return MIXED if row.regressions else STEADY + + +def _perf_verdict(row: Row, previous: Optional[PerformanceResult]) -> str: + if row.value is None: + return NO_DATA + if previous is None: + return FIRST + if previous.checkpoints_passed is True and row.perf.checkpoints_passed is False: + return REGRESSED + if not row.previous: # the last run measured nothing + return IMPROVED if row.value else STEADY + change = (row.value - row.previous) / row.previous + if change <= -PERF_STEADY_BAND: + return REGRESSED + if change >= PERF_STEADY_BAND: + return IMPROVED + return STEADY + + +def _result_row(session: Session, run: Run, kind: str) -> Row: + run = queries.get_run(session, run.id) + previous = queries.previous_run(session, run) + history_runs = [r for r, _ in queries.series_history(session, run, HISTORY_RUNS)] + counts = queries.status_counts(session, [r.id for r in history_runs]) + history = [queries.pass_rate(counts[r.id]) for r in history_runs] + row = Row( + run=run, + kind=kind, + verdict=FIRST, + value=history[-1], + previous=None, + previous_run=previous, + history=history, + counts=counts[run.id], + ) + if previous is not None: + base = queries.get_run(session, previous.id) + diff = queries.diff_runs(base, run) + row.previous = queries.pass_rate( + queries.status_counts(session, [base.id])[base.id] + ) + row.regressions = diff.count("regression") + row.fixed = diff.count("fixed") + row.changes = _changes_by_agent(diff) + row.flags = trapi_flag(base, run) + result_flags(base, run) + row.verdict = _result_verdict(row) + return row + + +def _perf_rows(session: Session, run: Run) -> list[Row]: + run = queries.get_run(session, run.id) + rows = [] + series = queries.performance_series(session, run, HISTORY_RUNS) + for perf, history in sorted( + series, key=lambda s: queries.agent_sort_key(_perf_name(s[0])) + ): + previous_run, previous = history[-2] if len(history) > 1 else (None, None) + row = Row( + run=run, + kind=PERFORMANCE, + verdict=FIRST, + value=perf.max_sustainable_concurrency, + previous=previous.max_sustainable_concurrency if previous else None, + previous_run=previous_run, + history=[p.max_sustainable_concurrency for _, p in history], + perf=perf, + ) + row.flags = trapi_flag(previous_run, run) + performance_flags( + run, perf, previous + ) + row.verdict = _perf_verdict(row, previous) + rows.append(row) + return rows + + +def rows_for(session: Session, runs: Iterable[Run]) -> list[Row]: + """Board rows for these runs: acceptance, then pathfinder, then + performance, each kind in the order its runs started.""" + runs = sorted(runs, key=lambda r: r.started_at) + kinds = run_kinds(session, [r.id for r in runs]) + rows = [] + for kind in KINDS: + of_kind = [] + for run in runs: + if kinds[run.id] != kind: + continue + if kind == PERFORMANCE: + of_kind.extend(_perf_rows(session, run)) + else: + of_kind.append(_result_row(session, run, kind)) + if kind == PERFORMANCE: + # the ARS first, then the ARAs, whatever order they ran in + of_kind.sort(key=lambda row: queries.agent_sort_key(_perf_name(row.perf))) + rows.extend(of_kind) + return rows + + +# --- cycles ------------------------------------------------------------------------- + + +@dataclass +class Cycle: + cycle_id: uuid.UUID + runs: list[Run] + rows: list[Row] + + @property + def env(self) -> Optional[str]: + envs = {r.env for r in self.runs} + return envs.pop() if len(envs) == 1 else None + + @property + def started_at(self) -> datetime: + return min(r.started_at for r in self.runs) + + @property + def ended_at(self) -> Optional[datetime]: + ends = [r.ended_at for r in self.runs] + return None if None in ends else max(ends) + + @property + def flags(self) -> list[Flag]: + return [flag for row in self.rows for flag in row.flags] + + +def get_cycle(session: Session, cycle_id: uuid.UUID) -> Optional[Cycle]: + runs = list( + session.scalars( + select(Run).where(Run.cycle_id == cycle_id).order_by(Run.started_at) + ) + ) + if not runs: + return None + return Cycle(cycle_id, runs, rows_for(session, runs)) + + +def recent_cycles( + session: Session, limit: int = 20 +) -> list[tuple[uuid.UUID, Optional[str], datetime]]: + """(cycle id, env, start) of the latest cycles, newest first.""" + from sqlalchemy import func + + rows = session.execute( + select(Run.cycle_id, func.min(Run.env), func.min(Run.started_at)) + .where(Run.cycle_id.is_not(None)) + .group_by(Run.cycle_id) + .order_by(func.min(Run.started_at).desc()) + .limit(limit) + ) + return [tuple(row) for row in rows] + + +# --- the digest --------------------------------------------------------------------- + + +@dataclass +class EnvDigest: + env: Optional[str] + rows: list[Row] + cycle_ids: list[uuid.UUID] + + def cell(self, kind: str) -> list[Row]: + return [row for row in self.rows if row.kind == kind] + + @property + def flags(self) -> list[Flag]: + return [flag for row in self.rows for flag in row.flags] + + +@dataclass +class Digest: + since: datetime + until: datetime + envs: list[EnvDigest] + + +def _env_key(env: Optional[str]): + if env is None: + return (2, 0, "") + if env in ENV_ORDER: + return (0, ENV_ORDER.index(env), env) + return (1, 0, env) + + +def digest(session: Session, until: Optional[datetime] = None, days: int = 7) -> Digest: + """Every environment's latest finished run of each test over ``days``. + + Full runs only: a run of one query type is a slice, and an acceptance run + against an overridden target isn't the environment's. Performance runs + always name their target, so they're kept, one row per service. + """ + until = until or datetime.now(timezone.utc) + since = until - timedelta(days=days) + runs = list( + session.scalars( + select(Run) + .where( + Run.started_at >= since, + Run.started_at <= until, + Run.ended_at.is_not(None), + Run.query_type.is_(None), + ) + .order_by(Run.started_at.desc()) + ) + ) + kinds = run_kinds(session, [r.id for r in runs]) + latest: dict = {} # (env, kind, suite, perf key) -> run + perf_hosts = _perf_hosts( + session, [r.id for r in runs if kinds[r.id] == PERFORMANCE] + ) + for run in runs: # newest first, so the first one seen is the latest + kind = kinds[run.id] + if kind == PERFORMANCE: + for host in perf_hosts.get(run.id, []): + latest.setdefault((run.env, kind, run.suite, host), run) + elif run.target is None: + latest.setdefault((run.env, kind, run.suite, None), run) + + by_env: dict = defaultdict(dict) + for (env, kind, suite, host), run in latest.items(): + by_env[env].setdefault(run.id, (run, set()))[1].add(host) + envs = [] + for env in sorted(by_env, key=_env_key): + picked = by_env[env] + rows = [] + for row in rows_for(session, [run for run, _ in picked.values()]): + # a performance run may have measured other hosts too; keep only + # the ones this run is the latest for + if row.perf is not None and row.perf.host not in picked[row.run.id][1]: + continue + rows.append(row) + cycle_ids = sorted( + {run.cycle_id for run, _ in picked.values() if run.cycle_id}, + key=str, + ) + envs.append(EnvDigest(env, rows, cycle_ids)) + return Digest(since, until, envs) + + +def _perf_hosts(session: Session, run_ids: list[uuid.UUID]) -> dict: + hosts: dict = defaultdict(list) + if not run_ids: + return hosts + for run_id, host in session.execute( + select(PerformanceResult.run_id, PerformanceResult.host).where( + PerformanceResult.run_id.in_(run_ids) + ) + ): + hosts[run_id].append(host) + return hosts + + +# --- JSON --------------------------------------------------------------------------- + + +def _asset_json(ref: AssetRef) -> dict: + return { + "test_case_id": ref.test_case_id, + "asset_id": ref.asset_id, + "name": ref.name, + } + + +def flag_json(flag: Flag) -> dict: + return { + "kind": flag.kind, + "who": flag.who, + "text": flag.text, + "run_id": str(flag.run_id), + "assets": [_asset_json(a) for a in flag.assets], + } + + +def row_json(row: Row) -> dict: + return { + "run_id": str(row.run.id), + "number": row.run.number, + "suite": row.run.suite, + "env": row.run.env, + "kind": row.kind, + "label": row.label, + "verdict": row.verdict, + "verdict_label": VERDICT_LABELS[row.verdict], + "value": row.value, + "previous": row.previous, + "previous_run_id": str(row.previous_run.id) if row.previous_run else None, + "value_text": row.value_text, + "unit": row.unit, + "change_text": row.change_text, + "detail_text": row.detail_text, + "trapi_version": row.run.trapi_version, + "regressions": row.regressions, + "fixed": row.fixed, + "started_at": row.run.started_at.isoformat(), + "host": row.perf.host if row.perf is not None else None, + "flags": [flag_json(f) for f in row.flags], + "changes": [ + { + "agent": c.agent, + "regressions": [_asset_json(a) for a in c.regressions], + "fixed": [_asset_json(a) for a in c.fixed], + } + for c in row.changes + ], + } + + +def cycle_json(cycle: Cycle) -> dict: + return { + "cycle_id": str(cycle.cycle_id), + "env": cycle.env, + "started_at": cycle.started_at.isoformat(), + "ended_at": cycle.ended_at.isoformat() if cycle.ended_at else None, + "rows": [row_json(row) for row in cycle.rows], + } + + +def digest_json(d: Digest) -> dict: + return { + "since": d.since.isoformat(), + "until": d.until.isoformat(), + "envs": [ + { + "env": e.env, + "cycle_ids": [str(c) for c in e.cycle_ids], + "rows": [row_json(row) for row in e.rows], + } + for e in d.envs + ], + } diff --git a/radiator/db.py b/radiator/db.py new file mode 100644 index 0000000..d272661 --- /dev/null +++ b/radiator/db.py @@ -0,0 +1,9 @@ +"""Database engine and sessions.""" + +from sqlalchemy import create_engine +from sqlalchemy.orm import Session, sessionmaker + + +def make_sessionmaker(database_url: str) -> sessionmaker[Session]: + engine = create_engine(database_url, pool_pre_ping=True) + return sessionmaker(engine, expire_on_commit=False) diff --git a/radiator/demo.py b/radiator/demo.py new file mode 100644 index 0000000..713708e --- /dev/null +++ b/radiator/demo.py @@ -0,0 +1,479 @@ +"""Fill a development database with made-up runs. + + RADIATOR_DATABASE_URL=... python -m radiator.demo + +Everything here is fake: it exists to develop and screenshot the dashboard +against something shaped like real history (regressions, fixes, ranks that +drift, a flaky agent, performance sweeps). Never point this at production. +""" + +import random +import uuid +from datetime import datetime, timedelta, timezone + +import radiator_schema as schema +from radiator.config import Settings # noqa: F401 (documents the env var) +from radiator.db import make_sessionmaker +from radiator.ingest import ingest_payload + +AGENTS = ["ars", "shepherd-aragorn", "shepherd-arax", "shepherd-bte"] + +PAIRS = [ + ("Metformin", "type 2 diabetes mellitus", "CHEBI:6801", "MONDO:0005148"), + ("Imatinib", "chronic myelogenous leukemia", "CHEBI:45783", "MONDO:0011996"), + ("Albuterol", "asthma", "CHEBI:2549", "MONDO:0004979"), + ("Levothyroxine", "hypothyroidism", "CHEBI:18332", "MONDO:0005420"), + ("Sumatriptan", "migraine disorder", "CHEBI:10650", "MONDO:0005277"), + ("Lisinopril", "hypertension", "CHEBI:43755", "MONDO:0005044"), + ("Methotrexate", "rheumatoid arthritis", "CHEBI:44185", "MONDO:0008383"), + ("Donepezil", "Alzheimer disease", "CHEBI:53289", "MONDO:0004975"), + ("Omeprazole", "gastroesophageal reflux", "CHEBI:7772", "MONDO:0007186"), + ("Sertraline", "major depressive disorder", "CHEBI:9123", "MONDO:0002009"), + ("Allopurinol", "gout", "CHEBI:40279", "MONDO:0005393"), + ("Tamoxifen", "breast carcinoma", "CHEBI:41774", "MONDO:0004989"), + ("Warfarin", "venous thromboembolism", "CHEBI:10033", "MONDO:0005147"), + ("Hydroxychloroquine", "lupus erythematosus", "CHEBI:5801", "MONDO:0004670"), + ("Isoniazid", "tuberculosis", "CHEBI:6030", "MONDO:0018076"), + ("Riluzole", "amyotrophic lateral sclerosis", "CHEBI:8863", "MONDO:0004976"), + ("Ivermectin", "onchocerciasis", "CHEBI:6078", "MONDO:0005301"), + ("Montelukast", "allergic rhinitis", "CHEBI:50730", "MONDO:0011786"), + ("Pyridostigmine", "myasthenia gravis", "CHEBI:8665", "MONDO:0009688"), + ("Dapsone", "leprosy", "CHEBI:4325", "MONDO:0005124"), + ("Insulin glargine", "type 1 diabetes mellitus", "CHEBI:5931", "MONDO:0005147"), + ("Penicillamine", "Wilson disease", "CHEBI:7959", "MONDO:0010200"), + ("Ursodiol", "primary biliary cholangitis", "CHEBI:9907", "MONDO:0005388"), + ("Nitisinone", "tyrosinemia type I", "CHEBI:50378", "MONDO:0010161"), +] +GENES = [ + ("Bortezomib", "PSMB5", "CHEBI:52717", "NCBIGene:5693"), + ("Erlotinib", "EGFR", "CHEBI:114785", "NCBIGene:1956"), + ("Simvastatin", "HMGCR", "CHEBI:9150", "NCBIGene:3156"), + ("Celecoxib", "PTGS2", "CHEBI:41423", "NCBIGene:5743"), + ("Rapamycin", "MTOR", "CHEBI:9168", "NCBIGene:2475"), + ("Vemurafenib", "BRAF", "CHEBI:63637", "NCBIGene:673"), + ("Tofacitinib", "JAK3", "CHEBI:71200", "NCBIGene:3718"), + ("Olaparib", "PARP1", "CHEBI:83766", "NCBIGene:142"), +] +NEVER = [ + ("Ethanol", "type 2 diabetes mellitus", "CHEBI:16236", "MONDO:0005148"), + ("Caffeine", "chronic myelogenous leukemia", "CHEBI:27732", "MONDO:0011996"), + ("Sodium chloride", "hypertension", "CHEBI:26710", "MONDO:0005044"), + ("Glucose", "migraine disorder", "CHEBI:17234", "MONDO:0005277"), +] + + +def build_assets(): + assets = [] + expectations = [ + "TopAnswer", + "TopAnswer", + "Acceptable", + "Acceptable", + "BadButForgivable", + ] + for i, (drug, disease, chem, mondo) in enumerate(PAIRS): + assets.append( + dict( + test_case_id=f"TestCase_{i // 4 + 1}", + asset_id=f"Asset_{i + 1}", + name=f"{drug} treats {disease}", + expected_output=expectations[i % 5], + predicate="biolink:treats", + input_curie=mondo, + output_curie=chem, + difficulty=random.random(), + ) + ) + for j, (drug, gene, chem, ncbi) in enumerate(GENES): + assets.append( + dict( + test_case_id=f"TestCase_{20 + j // 4}", + asset_id=f"Asset_{100 + j}", + name=f"{drug} affects {gene}", + expected_output="TopAnswer" if j % 3 else "Acceptable", + predicate="biolink:affects", + input_curie=ncbi, + output_curie=chem, + difficulty=random.random(), + ) + ) + for k, (chem_name, disease, chem, mondo) in enumerate(NEVER): + assets.append( + dict( + test_case_id="TestCase_30", + asset_id=f"Asset_{200 + k}", + name=f"{chem_name} treats {disease}", + expected_output="NeverShow", + predicate="biolink:treats", + input_curie=mondo, + output_curie=chem, + difficulty=0.2, + ) + ) + return assets + + +AGENT_SKILL = { + "ars": 0.78, + "shepherd-aragorn": 0.7, + "shepherd-arax": 0.66, + "shepherd-bte": 0.55, +} + + +def agent_result(agent, spec, run_index, n_runs, rng, skill_offset=0.0): + skill = AGENT_SKILL[agent] + skill_offset + # aragorn regresses for a stretch, then a fix lands + if agent == "shepherd-aragorn" and n_runs * 0.45 < run_index < n_runs * 0.7: + skill -= 0.25 + # bte gets steadily better + if agent == "shepherd-bte": + skill += 0.25 * run_index / n_runs + # the shepherds are queried directly, so their response times are measured + response_time_s = ( + None if agent == "ars" else round(rng.lognormvariate(1.6, 0.35), 2) + ) + roll = rng.random() + if roll < 0.03: + return schema.AgentResult( + agent=agent, status="FAILED", message="Timed out", http_status=598 + ) + if roll < 0.06 and agent != "ars": + return schema.AgentResult( + agent=agent, + status="ERROR", + message="Status code: 500", + http_status=500, + response_time_s=response_time_s, + ) + n_results = int(rng.lognormvariate(5.2, 0.7)) + if rng.random() < 0.05: + return schema.AgentResult( + agent=agent, + status="NO_RESULTS", + message="No results", + http_status=200, + n_results=0, + found=False, + response_time_s=response_time_s, + ) + expected = spec["expected_output"] + good = rng.random() < skill + 0.25 - spec["difficulty"] * 0.4 + if expected is None: # pathfinder: did the expected path come back? + status = "PASSED" if good else "FAILED" + return schema.AgentResult( + agent=agent, + status=status, + http_status=200, + found=good, + n_results=n_results, + response_time_s=response_time_s, + expected_nodes_found="3/3" if good else f"{rng.randint(0, 2)}/3", + pk=str(uuid.UUID(int=rng.getrandbits(128))), + ) + if expected == "NeverShow": + found = not good + rank = rng.randint(1, n_results) if found else None + status = "PASSED" if not found else "FAILED" + else: + found = good or rng.random() < 0.5 + if found: + base = 1 + spec["difficulty"] * 40 + rank = max(1, int(rng.gauss(base if good else base * 4 + 30, 5))) + rank = min(rank, n_results) + else: + rank = None + limit = { + "TopAnswer": 30, + "Acceptable": n_results // 2, + "BadButForgivable": n_results, + }[expected] + status = "PASSED" if found and rank is not None and rank <= limit else "FAILED" + score = ( + None + if rank is None + else round(max(0.05, 1 - rank / max(n_results, 1)) * rng.uniform(0.85, 1.0), 3) + ) + return schema.AgentResult( + agent=agent, + status=status, + http_status=200, + found=found, + rank=rank, + score=score, + n_results=n_results, + response_time_s=response_time_s, + pk=str(uuid.UUID(int=rng.getrandbits(128))), + ) + + +PATHS = [ + ("Imatinib", "chronic myelogenous leukemia", "CHEBI:45783", "MONDO:0011996"), + ("Metformin", "polycystic ovary syndrome", "CHEBI:6801", "MONDO:0008487"), + ("Sildenafil", "pulmonary hypertension", "CHEBI:9139", "MONDO:0005149"), + ("Thalidomide", "multiple myeloma", "CHEBI:9513", "MONDO:0009693"), + ("Aspirin", "colorectal cancer", "CHEBI:15365", "MONDO:0005575"), + ("Propranolol", "infantile hemangioma", "CHEBI:8499", "MONDO:0006500"), + ("Rituximab", "pemphigus vulgaris", "CHEBI:64357", "MONDO:0006570"), + ("Dexamethasone", "COVID-19", "CHEBI:41879", "MONDO:0100096"), + ("Baricitinib", "alopecia areata", "CHEBI:95341", "MONDO:0005343"), + ("Tocilizumab", "giant cell arteritis", "CHEBI:64359", "MONDO:0008798"), + ("Valproic acid", "bipolar disorder", "CHEBI:39867", "MONDO:0004985"), + ("Topiramate", "obesity", "CHEBI:63631", "MONDO:0011122"), +] + + +def build_paths(): + return [ + dict( + test_case_id=f"PathCase_{i // 4 + 1}", + asset_id=f"Path_{i + 1}", + name=f"{drug} to {disease}", + expected_output=None, + input_curie=chem, + output_curie=mondo, + difficulty=random.random() * 0.8, + ) + for i, (drug, disease, chem, mondo) in enumerate(PATHS) + ] + + +# How each environment differs: dev is the roughest, prod the steadiest. +ENV_SKILL = {"dev": -0.1, "ci": 0.0, "test": 0.02, "prod": 0.05} +# The rotation: one environment a day, Monday to Thursday. +ENV_DAY = {"dev": 0, "ci": 1, "test": 2, "prod": 3} +# The share of agent results that come out differently from last week. +CHURN = 0.06 +ARS_CONCURRENCY = {"dev": 9.0, "ci": 14.0, "test": 13.0, "prod": 16.0} + + +def ars_url(env): + return ( + "https://ars.transltr.io" if env == "prod" else f"https://ars.{env}.transltr.io" + ) + + +def demo_runs(weeks=8, seed=7): + """Weekly run cycles in each environment (acceptance, pathfinder, ARS + performance), plus a Sunday performance sweep of the ARAs on ci. + + The latest week carries a few oddities for the flags to find: arax stops + returning results on some ci assets and bte gets slower there, aragorn + ranks a couple of ci answers much lower while still passing them, dev's + aragorn starts erroring, and the ARS on test loses a third of its + concurrency. + """ + rng = random.Random(seed) + random.seed(seed) + assets = build_assets() + paths = build_paths() + now = datetime.now(timezone.utc).replace(minute=0, second=0, microsecond=0) + this_monday = (now - timedelta(days=now.weekday())).replace(hour=6) + first_monday = this_monday - timedelta(weeks=weeks - 1) + payloads = [] + + def results_run(env, kind, suite, key, started, week, cycle_id, quirks=()): + run = schema.RunCreate( + run_id=uuid.uuid5(uuid.NAMESPACE_URL, f"demo:{suite}:{env}:{key}"), + suite=suite, + env=env, + harness_version="0.8.0", + tests_source="https://github.com/NCATSTranslator/Tests/archive/refs/heads/main.zip", + started_at=started, + cycle_id=cycle_id, + ) + latest = week == weeks - 1 + results = [] + counts = {} + specs = assets if kind == "acceptance" else paths + for n, spec in enumerate(specs): + if "skip_all" in quirks: + agents = [schema.AgentResult(agent=a, status="SKIPPED") for a in AGENTS] + else: + agents = [] + for a in AGENTS: + # Mostly the same answer as last week: an agent's outcome + # on an asset is seeded by the asset, and only now and + # then (or as its skill drifts) does it change. + stable = random.Random(f"{seed}:{env}:{a}:{spec['asset_id']}") + pick = rng if rng.random() < CHURN else stable + agents.append( + agent_result(a, spec, week, weeks, pick, ENV_SKILL[env]) + ) + for i, agent in enumerate(agents): + if "arax_no_results" in quirks and agent.agent == "shepherd-arax": + if latest and n < 5 and agent.status != "SKIPPED": + agents[i] = schema.AgentResult( + agent=agent.agent, + status="NO_RESULTS", + message="No results", + http_status=200, + n_results=0, + found=False, + response_time_s=agent.response_time_s, + ) + elif n < 5: + agents[i] = agent.model_copy( + update={ + "status": schema.Status.PASSED, + "n_results": 180, + "found": True, + } + ) + if "bte_slower" in quirks and agent.agent == "shepherd-bte": + if agent.response_time_s is not None: + agents[i] = agent.model_copy( + update={"response_time_s": agent.response_time_s * 4 + 6} + ) + if "aragorn_rank" in quirks and agent.agent == "shepherd-aragorn": + if spec["expected_output"] == "BadButForgivable" and n < 12: + agents[i] = agent.model_copy( + update={ + "status": schema.Status.PASSED, + "found": True, + "rank": (52 + n) if latest else 4, + "n_results": 400, + "score": 0.2 if latest else 0.9, + } + ) + if "aragorn_errors" in quirks and agent.agent == "shepherd-aragorn": + if latest and n % 5 == 0: + agents[i] = schema.AgentResult( + agent=agent.agent, + status="ERROR", + message="Status code: 502", + http_status=502, + ) + elif not latest and agent.status == "ERROR": + agents[i] = agent.model_copy( + update={"status": schema.Status.FAILED, "http_status": 200} + ) + overall = agents[0].status + counts[overall] = counts.get(overall, 0) + 1 + fields = {k: v for k, v in spec.items() if k != "difficulty"} + results.append( + schema.AssetResult( + kind=kind, + status=overall, + agents=agents, + parent_pk=str(uuid.UUID(int=rng.getrandbits(128))), + **fields, + ) + ) + minutes = rng.randint(42, 58) if kind == "acceptance" else rng.randint(8, 15) + payloads.append( + schema.RunPayload( + run=run, + results=results, + finish=schema.RunFinish( + ended_at=started + timedelta(minutes=minutes), counts=counts + ), + ) + ) + return run.started_at + timedelta(minutes=minutes + 2) + + def performance_run(env, key, started, host, target, msc, cycle_id=None): + passed = ( + msc >= ARS_CONCURRENCY.get(env, 10) * 0.8 if target == "ars" else msc >= 5 + ) + run = schema.RunCreate( + run_id=uuid.uuid5(uuid.NAMESPACE_URL, f"demo:perf:{env}:{key}:{target}"), + suite="performance_tests", + env=env, + target=target, + target_url=host, + harness_version="0.8.0", + started_at=started, + cycle_id=cycle_id, + ) + perf = schema.PerformanceResult( + test_case_id="Perf_1", + asset_id="Perf_mixed", + host=host, + helmsdeep_target=target, + profile="mixed", + status="PASSED" if passed else "FAILED", + exit_code=0, + max_sustainable_concurrency=msc, + checkpoints_passed=passed, + summary={"max_sustainable_concurrency": msc, "checkpoints_passed": passed}, + ) + payloads.append( + schema.RunPayload( + run=run, + performance=[perf], + finish=schema.RunFinish(ended_at=started + timedelta(minutes=68)), + ) + ) + + quirks = { + "ci": ("arax_no_results", "bte_slower", "aragorn_rank"), + "dev": ("aragorn_errors",), + } + for week in range(weeks): + for env, day in ENV_DAY.items(): + started = first_monday + timedelta(weeks=week, days=day) + if started > now: + continue + cycle_id = uuid.uuid5(uuid.NAMESPACE_URL, f"demo:cycle:{env}:{week}") + env_quirks = quirks.get(env, ()) + # one dev week where nothing could run + if env == "dev" and week == weeks - 3: + env_quirks = ("skip_all",) + # bte is only slower this week + if "bte_slower" in env_quirks and week != weeks - 1: + env_quirks = tuple(q for q in env_quirks if q != "bte_slower") + t = results_run( + env, + "acceptance", + "sprint_4_tests", + week, + started, + week, + cycle_id, + env_quirks, + ) + t = results_run( + env, "pathfinder", "pathfinder_tests", week, t, week, cycle_id + ) + msc = rng.gauss(ARS_CONCURRENCY[env], 0.5) + if env == "test" and week == weeks - 1: + msc *= 0.66 + performance_run( + env, week, t, ars_url(env), "ars", round(max(1.0, msc), 1), cycle_id + ) + + # a Sunday sweep of the ARAs on ci, outside the cycles + for week in range(weeks): + sweep_start = first_monday + timedelta(weeks=week, days=-1, hours=-4) + for position, (target, base) in enumerate( + [("aragorn", 9.0), ("arax", 6.5), ("bte", 11.0)] + ): + trend = -0.3 * week if target == "arax" else 0 + msc = round(max(1.0, rng.gauss(base + trend, 0.6)), 1) + performance_run( + "ci", + f"sweep{week}", + sweep_start + timedelta(minutes=70 * position), + f"https://{target}.ci.transltr.io", + target, + msc, + ) + return payloads + + +def main(): + import os + + sessionmaker = make_sessionmaker(os.environ["RADIATOR_DATABASE_URL"]) + # in the order they ran, so their run numbers follow time + payloads = sorted(demo_runs(), key=lambda p: p.run.started_at) + with sessionmaker() as session: + for payload in payloads: + ingest_payload(session, payload) + session.commit() + print(f"Loaded {len(payloads)} demo runs.") + + +if __name__ == "__main__": + main() diff --git a/radiator/grid.py b/radiator/grid.py new file mode 100644 index 0000000..36dcec8 --- /dev/null +++ b/radiator/grid.py @@ -0,0 +1,383 @@ +"""The acceptance run grid: every run of a suite, by environment, over time. + +A lane per environment, time across. Each run is a card (number, start time, +pass rate and its change, passed / failed / skipped) whose trend compares its +pass rate with the previous run in the same environment: UP, DOWN, FLAT, or +BASE for the first one shown. Excluded environments (dev, by default) are drawn +greyed out, without a trend. Runs too close in time to sit side by side stack +downward, and each run is joined to the one before it in its lane. + +The layout is computed here and drawn as SVG for the dashboard (cards link +to their runs, colors follow the page's theme). +""" + +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from typing import Iterable, Optional + +from markupsafe import Markup, escape +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from radiator.models import AssetResult, Run + +# Lanes in this order; any other environment after these, alphabetically, and +# runs with no environment last. +ENV_ORDER = ["dev", "ci", "test", "prod"] +NOT_PASSING = ("FAILED", "NO_RESULTS", "ERROR") + +UP, DOWN, FLAT, BASE, EXCLUDED, NO_DATA = "up", "down", "flat", "base", "excl", "nodata" +TREND_LABELS = { + UP: "UP", + DOWN: "DOWN", + FLAT: "FLAT", + BASE: "BASE", + EXCLUDED: "EXCL", + NO_DATA: "N/A", +} + +# Geometry, in px of the drawing. +CARD_W, CARD_H = 188, 92 +GAP_X, GAP_Y = 14, 14 +LANE_PAD = 20 +LABEL_W = 150 +RIGHT_PAD = 24 +AXIS_H = 40 + + +@dataclass +class GridRun: + run: Run + passed: int + failed: int + skipped: int + rate: Optional[float] + trend: str = BASE + delta: Optional[float] = None # percentage points vs the previous run + + +@dataclass +class Card: + item: GridRun + x: float # top left + y: float + level: int + + +@dataclass +class Lane: + env: Optional[str] + excluded: bool + y: float + height: float + cards: list[Card] = field(default_factory=list) + + @property + def label(self) -> str: + return (self.env or "no environment").upper() + + +@dataclass +class Link: + x1: float + y1: float + x2: float + y2: float + trend: str + + +@dataclass +class Grid: + suite: str + since: datetime + until: datetime + width: float + height: float # lanes + time axis + lanes: list[Lane] + links: list[Link] + ticks: list[tuple[float, str]] + + @property + def empty(self) -> bool: + return not self.lanes + + +# --- data ------------------------------------------------------------------------ + + +def grid_runs( + session: Session, + suite: str, + since: datetime, + until: datetime, + excluded_envs: Iterable[str] = ("dev",), +) -> list[GridRun]: + """The suite's finished, full acceptance runs in the window, oldest first, + each with its trend against the previous run in its environment. + + "Full" leaves out runs with a target override or a single query type, + which measure something else. The run before the window is fetched too, + so the first card shown still has a trend rather than starting over. + """ + full = [ + Run.suite == suite, + Run.target.is_(None), + Run.query_type.is_(None), + Run.ended_at.is_not(None), + Run.started_at <= until, + ] + runs = list( + session.scalars( + select(Run).where(*full, Run.started_at >= since).order_by(Run.started_at) + ) + ) + # one run per environment from just before the window, as a baseline + envs = {run.env for run in runs} + baselines = [] + for env in envs: + before = session.scalar( + select(Run) + .where(*full, Run.env.is_not_distinct_from(env), Run.started_at < since) + .order_by(Run.started_at.desc()) + .limit(1) + ) + if before is not None: + baselines.append(before) + + all_runs = baselines + runs + counts: dict = {run.id: {} for run in all_runs} + if all_runs: + rows = session.execute( + select(AssetResult.run_id, AssetResult.status, func.count()) + .where(AssetResult.run_id.in_(list(counts))) + .group_by(AssetResult.run_id, AssetResult.status) + ) + for run_id, status, n in rows: + counts[run_id][status] = n + + excluded = {e.lower() for e in excluded_envs} + previous_rate: dict = {} + items = [] + for run in sorted(all_runs, key=lambda r: r.started_at): + c = counts[run.id] + if not c: + continue # no acceptance results (eg a performance run) + passed = c.get("PASSED", 0) + failed = sum(c.get(s, 0) for s in NOT_PASSING) + skipped = c.get("SKIPPED", 0) + rate = passed / (passed + failed) if passed + failed else None + item = GridRun(run, passed, failed, skipped, rate) + prev = previous_rate.get(run.env) + if (run.env or "").lower() in excluded: + item.trend = EXCLUDED + elif rate is None: + item.trend = NO_DATA + elif prev is None: + item.trend = BASE + else: + item.delta = (rate - prev) * 100 + # FLAT when equal to a tenth of a point + if round(rate * 1000) == round(prev * 1000): + item.trend = FLAT + else: + item.trend = UP if rate > prev else DOWN + if rate is not None: + previous_rate[run.env] = rate + if run.started_at >= since: + items.append(item) + return items + + +# --- layout ---------------------------------------------------------------------- + + +def _env_key(env: Optional[str]): + if env in ENV_ORDER: + return (0, ENV_ORDER.index(env), "") + return (1 if env else 2, 0, env or "") + + +def _ticks(since: datetime, until: datetime) -> list[datetime]: + """At midnight: daily for a short window, every other day for a few + weeks, and on Mondays beyond that.""" + span_days = (until - since).total_seconds() / 86400 + step = 1 if span_days <= 10 else 2 if span_days <= 21 else 7 + first = since.replace(hour=0, minute=0, second=0, microsecond=0) + if first < since: + first += timedelta(days=1) + if step == 7: + first += timedelta(days=(7 - first.weekday()) % 7) + ticks = [] + t = first + while t <= until: + ticks.append(t) + t += timedelta(days=step) + return ticks + + +def layout( + suite: str, + items: list[GridRun], + since: datetime, + until: datetime, + width: float, + excluded_envs: Iterable[str] = ("dev",), +) -> Grid: + plot_left, plot_right = LABEL_W, width - RIGHT_PAD + span = (until - since).total_seconds() or 1 + + def center_x(t: datetime) -> float: + frac = min(max((t - since).total_seconds() / span, 0), 1) + return plot_left + CARD_W / 2 + frac * (plot_right - plot_left - CARD_W) + + def time_x(t: datetime) -> float: + frac = (t - since).total_seconds() / span + return plot_left + CARD_W / 2 + frac * (plot_right - plot_left - CARD_W) + + excluded = {e.lower() for e in excluded_envs} + by_env: dict = {} + for item in items: + by_env.setdefault(item.run.env, []).append(item) + + lanes, links = [], [] + y = 0.0 + for env in sorted(by_env, key=_env_key): + lane_items = sorted(by_env[env], key=lambda i: i.run.started_at) + # greedy stacking: each card on the highest level it fits on + level_right: list[float] = [] + cards = [] + for item in lane_items: + x = center_x(item.run.started_at) - CARD_W / 2 + level = next( + (i for i, right in enumerate(level_right) if x >= right + GAP_X), + len(level_right), + ) + if level == len(level_right): + level_right.append(x + CARD_W) + else: + level_right[level] = x + CARD_W + cards.append(Card(item, x, 0, level)) + levels = max(len(level_right), 1) + height = LANE_PAD * 2 + levels * CARD_H + (levels - 1) * GAP_Y + for card in cards: + card.y = y + LANE_PAD + card.level * (CARD_H + GAP_Y) + for a, b in zip(cards, cards[1:]): + links.append( + Link( + a.x + CARD_W / 2, + a.y + CARD_H / 2, + b.x + CARD_W / 2, + b.y + CARD_H / 2, + b.item.trend, + ) + ) + lanes.append(Lane(env, (env or "").lower() in excluded, y, height, cards)) + y += height + + ticks = [(time_x(t), t.strftime("%b %-d")) for t in _ticks(since, until)] + return Grid(suite, since, until, width, y + AXIS_H, lanes, links, ticks) + + +def build( + session: Session, + suite: str, + until: Optional[datetime] = None, + days: int = 35, + width: float = 1180, + excluded_envs: Iterable[str] = ("dev",), +) -> Grid: + """The grid for ``suite`` over the ``days`` before ``until`` (now).""" + until = until or datetime.now(timezone.utc) + since = until - timedelta(days=days) + items = grid_runs(session, suite, since, until, excluded_envs) + return layout(suite, items, since, until, width, excluded_envs) + + +# --- card text ----------------------------------------------------------------- + + +def card_lines(item: GridRun) -> tuple[str, str, str, str]: + """(number, start time, rate line, counts line).""" + rate = "all skipped" if item.rate is None else f"{item.rate * 100:.1f}%" + if item.delta is not None and item.trend != FLAT: + rate += f" {'▲' if item.delta > 0 else '▼'}{abs(item.delta):.1f} pts" + elif item.trend == FLAT: + rate += " no change" + counts = f"P {item.passed} · F {item.failed} · S {item.skipped}" + return ( + f"#{item.run.number}", + item.run.started_at.astimezone(timezone.utc).strftime("%m/%d %H:%M"), + rate, + counts, + ) + + +def card_tip(item: GridRun) -> str: + number, when, rate, _ = card_lines(item) + lines = [ + f"{number} · {item.run.env or 'no environment'} · {when} UTC", + f"{TREND_LABELS[item.trend]} · pass rate {rate}", + f"{item.passed} passed, {item.failed} not passing, {item.skipped} skipped", + ] + return "\n".join(lines) + + +# --- SVG --------------------------------------------------------------------------- + + +def to_svg(grid: Grid) -> Markup: + """The grid as SVG, colored by the page's CSS tokens; cards link to runs.""" + w, h = grid.width, grid.height + parts = [ + f'' + ] + for i, lane in enumerate(grid.lanes): + band = "band-alt" if i % 2 else "band" + parts.append( + f'' + ) + label_y = lane.y + lane.height / 2 + parts.append( + f'{escape(lane.label)}' + ) + if lane.excluded: + parts.append( + f'excluded' + ) + lanes_bottom = grid.height - AXIS_H + for x, label in grid.ticks: + parts.append( + f'' + f'{label}' + ) + for link in grid.links: + parts.append( + f'' + ) + for lane in grid.lanes: + for card in lane.cards: + parts.append(_svg_card(card)) + parts.append("") + return Markup("".join(parts)) + + +def _svg_card(card: Card) -> str: + item, x, y = card.item, card.x, card.y + number, when, rate, counts = card_lines(item) + trend = item.trend + label = TREND_LABELS[trend] + chip_w = 10 + 7.5 * len(label) + return ( + f'' + f'' + f'' + f'' + f'{label}' + f'{escape(number)}' + f'{escape(when)} UTC' + f'{escape(rate)}' + f'{escape(counts)}' + "" + ) diff --git a/radiator/history_png.py b/radiator/history_png.py new file mode 100644 index 0000000..708c31a --- /dev/null +++ b/radiator/history_png.py @@ -0,0 +1,324 @@ +"""A run's history as PNGs, for places that can't show the dashboard: chiefly +the harness's Slack report, which can't render SVG or reach pages behind the +login. + +* ``render``: pass rate by agent over the run's series. +* ``render_performance``: each of the run's services' max sustainable + concurrency over its series. + +Drawn with matplotlib in the dashboard's light palette, following the same +chart rules: one y-axis, step lines (a run's value holds until the next run), +hairline grid, each agent's fixed color, and this run's value written out, so +color never works alone. +""" + +import io +from dataclasses import dataclass +from datetime import datetime +from typing import Callable, Optional + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.dates as mdates # noqa: E402 +import matplotlib.pyplot as plt # noqa: E402 +from matplotlib.lines import Line2D # noqa: E402 +from matplotlib.ticker import FuncFormatter, MaxNLocator, MultipleLocator # noqa: E402 + +from radiator import queries # noqa: E402 +from radiator.charts import AGENT_COLOR_SLOTS # noqa: E402 + +# The light-mode values of the CSS tokens in static/style.css. +SURFACE = "#fcfcfb" +INK = "#0b0b0b" +INK_2 = "#52514e" +MUTED = "#6f6d68" +GRID = "#e1e0d9" +AXIS = "#c3c2b7" +STATUS_FAILED = "#d03b3b" +AGENT_HEX = { + 1: "#2a78d6", + 2: "#eb6834", + 3: "#1baf7a", + 4: "#eda100", + 5: "#e87ba4", + 6: "#008300", + 7: "#4a3aa7", + 8: "#e34948", +} + + +def agent_hex(agent: Optional[str]) -> str: + return AGENT_HEX[AGENT_COLOR_SLOTS.get(agent or "", 8)] + + +@dataclass +class Line: + label: str + color: str + # one value per run, None where the run had none + values: list[Optional[float]] + # runs to flag on the line (eg missed checkpoints), as booleans per run + flagged: Optional[list[bool]] = None + + +def _step_segments(times, values): + """Unbroken stretches of values: a run with no value is a gap, rather + than the step before it carrying on across it.""" + segment = [] + for t, v in zip(times, values): + if v is None: + if segment: + yield segment + segment = [] + else: + segment.append((t, v)) + if segment: + yield segment + + +def _chart( + times: list[datetime], + lines: list[Line], + *, + title: str, + subtitle: str, + y_format: Callable[[float], str], + y_max: Optional[float] = None, + value_format: Callable[[float], str], + flag_label: str = "", + empty_text: str = "No data in this series", +) -> bytes: + plt.rcParams["font.family"] = "DejaVu Sans" + fig, ax = plt.subplots(figsize=(8, 4.2), dpi=200) + fig.patch.set_facecolor(SURFACE) + ax.set_facecolor(SURFACE) + + handles, labels = [], [] + any_flag = False + for line in lines: + drawn = False + for segment in _step_segments(times, line.values): + xs, ys = zip(*segment) + ax.plot( + xs, + ys, + drawstyle="steps-post", + color=line.color, + linewidth=1.6, + solid_joinstyle="miter", + zorder=3, + ) + drawn = True + if not drawn: + continue + if line.flagged: + flagged = [ + (t, v) + for t, v, f in zip(times, line.values, line.flagged) + if f and v is not None + ] + if flagged: + any_flag = True + fx, fy = zip(*flagged) + ax.scatter( + fx, + fy, + marker="x", + s=26, + color=STATUS_FAILED, + linewidths=1.4, + zorder=5, + ) + current = line.values[-1] + if current is not None: + # this run's value: the end dot, ringed in the surface color + ax.scatter( + [times[-1]], + [current], + s=34, + color=line.color, + edgecolors=SURFACE, + linewidths=1.6, + zorder=4, + ) + handles.append(Line2D([], [], color=line.color, linewidth=1.6)) + labels.append( + f"{line.label} {'–' if current is None else value_format(current)}" + ) + if any_flag: + handles.append( + Line2D( + [], + [], + marker="x", + linestyle="", + color=STATUS_FAILED, + markersize=5, + markeredgewidth=1.4, + ) + ) + labels.append(flag_label) + + # mark this run + ax.axvline(times[-1], color=AXIS, linewidth=0.8, zorder=1) + ax.annotate( + "this run", + xy=(1.0, 1.0), + xycoords="axes fraction", + xytext=(0, 4), + textcoords="offset points", + ha="right", + va="bottom", + fontsize=7, + color=MUTED, + ) + + top = y_max + if top is None: + values = [v for line in lines for v in line.values if v is not None] + top = max(values) * 1.15 if values and max(values) > 0 else 1 + ax.set_ylim(0, top) + if y_max == 1.0: + ax.yaxis.set_major_locator(MultipleLocator(0.25)) + else: + ax.yaxis.set_major_locator(MaxNLocator(nbins=5, integer=False)) + ax.yaxis.set_major_formatter(FuncFormatter(lambda v, _: y_format(v))) + ax.grid(axis="y", color=GRID, linewidth=0.6) + ax.set_axisbelow(True) + for side in ("top", "right", "left"): + ax.spines[side].set_visible(False) + ax.spines["bottom"].set_color(AXIS) + ax.spines["bottom"].set_linewidth(0.6) + ax.tick_params(colors=MUTED, labelsize=7, length=0, pad=4) + if len(times) > 1: + pad = (times[-1] - times[0]) * 0.02 + ax.set_xlim(times[0] - pad, times[-1] + pad) + ax.xaxis.set_major_locator(mdates.AutoDateLocator(minticks=3, maxticks=7)) + ax.xaxis.set_major_formatter(mdates.DateFormatter("%b %-d")) + + if len(lines) == 1 and len(handles) == 1: + # one series: the title already names it, so no legend; its value + # goes in the subtitle instead + current = lines[0].values[-1] + if current is not None: + subtitle += f" · this run {value_format(current)}" + handles, labels = [], [] + fig.text(0.06, 0.94, title, fontsize=10, fontweight="bold", color=INK) + fig.text(0.06, 0.885, subtitle, fontsize=7.5, color=INK_2) + if handles: + fig.legend( + handles, + labels, + loc="lower left", + bbox_to_anchor=(0.055, 0.0), + ncol=min(len(handles), 4), + frameon=False, + fontsize=7.5, + handlelength=1.4, + columnspacing=1.6, + labelcolor=INK_2, + ) + elif not any(v is not None for line in lines for v in line.values): + ax.text( + 0.5, + 0.5, + empty_text, + transform=ax.transAxes, + ha="center", + color=MUTED, + fontsize=8, + ) + + rows = (len(handles) + 3) // 4 if handles else 0 + fig.subplots_adjust(left=0.07, right=0.97, top=0.84, bottom=0.12 + 0.05 * rows) + out = io.BytesIO() + fig.savefig(out, format="png", facecolor=SURFACE) + plt.close(fig) + return out.getvalue() + + +def _runs_phrase(n: int, until: datetime) -> str: + return f"last {n} run{'s' if n != 1 else ''} to {until:%b %-d %Y %H:%M} UTC" + + +def render(run, history: list) -> bytes: + """Pass rate by agent. ``history`` is ``queries.series_history(...)``.""" + agents = sorted( + {agent for _, counts in history for agent in counts}, + key=queries.agent_sort_key, + ) + lines = [ + Line( + agent, + agent_hex(agent), + [queries.pass_rate(counts.get(agent, {})) for _, counts in history], + ) + for agent in agents + ] + extra = [] + if run.target: + extra.append(f"target {run.target}") + if run.query_type: + extra.append(f"{run.query_type} only") + title = f"{run.suite} · {run.env or 'no environment'}" + ( + f" · {', '.join(extra)}" if extra else "" + ) + return _chart( + [r.started_at for r, _ in history], + lines, + title=title, + subtitle=( + f"Pass rate by agent, {_runs_phrase(len(history), run.started_at)}" + " · skips excluded" + ), + y_format=lambda v: f"{v:.0%}", + y_max=1.0, + value_format=lambda v: f"{v:.0%}", + empty_text="No agent results in this series", + ) + + +def render_performance(run, series: list) -> bytes: + """Max sustainable concurrency for each service in the run. + + ``series`` is ``queries.performance_series(...)``: per service, its + history up to this run. Services can have different histories, so the + x-axis is the union of their runs. + """ + times = sorted({r.started_at for _, history in series for r, _ in history}) + lines = [] + for perf, history in series: + by_time = {r.started_at: p for r, p in history} + name = perf.helmsdeep_target or perf.host + lines.append( + Line( + name, + agent_hex(perf.helmsdeep_target), + [ + by_time[t].max_sustainable_concurrency if t in by_time else None + for t in times + ], + flagged=[ + t in by_time and by_time[t].checkpoints_passed is False + for t in times + ], + ) + ) + names = ", ".join(p.helmsdeep_target or p.host for p, _ in series) + profiles = sorted({p.profile or "default" for p, _ in series}) + longest = max((len(h) for _, h in series), default=0) + return _chart( + times, + lines, + title=f"{names} · {run.env or 'no environment'}", + subtitle=( + "Max sustainable concurrency (users within the SLO), " + f"{_runs_phrase(longest, run.started_at)} · profile {', '.join(profiles)}" + ), + y_format=lambda v: f"{v:g}", + value_format=lambda v: f"{v:.1f}", + flag_label="checkpoints missed", + empty_text="No stage met the SLO in these runs", + ) diff --git a/radiator/ingest.py b/radiator/ingest.py new file mode 100644 index 0000000..58d8372 --- /dev/null +++ b/radiator/ingest.py @@ -0,0 +1,105 @@ +"""Writing results: what the ingest API and the Zebrunner importer share. + +Every write is an upsert on the identity ``radiator_schema`` defines, so a +retried batch or a replayed run updates what's there instead of duplicating it. +An asset's agent rows are replaced wholesale on each upsert: the newest upload +is the whole truth about that asset in that run. +""" + +import uuid + +from sqlalchemy import delete, insert, update +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.orm import Session + +import radiator_schema as schema +from radiator.models import AgentResult, AssetResult, PerformanceResult, Run, utcnow + + +class RunNotFound(LookupError): + pass + + +def upsert_run(session: Session, run: schema.RunCreate) -> None: + values = run.model_dump(exclude={"run_id"}) + stmt = pg_insert(Run).values(id=run.run_id, counts={}, **values) + session.execute(stmt.on_conflict_do_update(index_elements=[Run.id], set_=values)) + + +def _require_run(session: Session, run_id: uuid.UUID) -> None: + if session.get(Run, run_id) is None: + raise RunNotFound(run_id) + + +def upsert_results( + session: Session, run_id: uuid.UUID, batch: schema.ResultBatch +) -> None: + _require_run(session, run_id) + # what the dashboard's live updates watch for + session.execute(update(Run).where(Run.id == run_id).values(updated_at=utcnow())) + for result in batch.results: + values = result.model_dump(exclude={"agents"}) + values["status"] = result.status.value + stmt = pg_insert(AssetResult).values(run_id=run_id, **values) + asset_result_id = session.execute( + stmt.on_conflict_do_update( + index_elements=[ + AssetResult.run_id, + AssetResult.test_case_id, + AssetResult.asset_id, + ], + set_=values, + ).returning(AssetResult.id) + ).scalar_one() + session.execute( + delete(AgentResult).where(AgentResult.asset_result_id == asset_result_id) + ) + if result.agents: + session.execute( + insert(AgentResult), + [ + { + **agent.model_dump(), + "status": agent.status.value, + "asset_result_id": asset_result_id, + } + for agent in result.agents + ], + ) + + for perf in batch.performance: + values = perf.model_dump() + values["status"] = perf.status.value + stmt = pg_insert(PerformanceResult).values(run_id=run_id, **values) + session.execute( + stmt.on_conflict_do_update( + index_elements=[ + PerformanceResult.run_id, + PerformanceResult.test_case_id, + PerformanceResult.asset_id, + PerformanceResult.host, + ], + set_=values, + ) + ) + + +def finish_run(session: Session, run_id: uuid.UUID, finish: schema.RunFinish) -> None: + _require_run(session, run_id) + session.execute( + update(Run) + .where(Run.id == run_id) + .values(ended_at=finish.ended_at, counts=finish.counts) + ) + + +def ingest_payload(session: Session, payload: schema.RunPayload) -> None: + """Write a whole run at once (the importer's path).""" + upsert_run(session, payload.run) + upsert_results( + session, + payload.run.run_id, + schema.ResultBatch(results=payload.results, performance=payload.performance), + ) + if payload.finish is not None: + finish_run(session, payload.run.run_id, payload.finish) diff --git a/radiator/migrations/env.py b/radiator/migrations/env.py new file mode 100644 index 0000000..e8f5000 --- /dev/null +++ b/radiator/migrations/env.py @@ -0,0 +1,51 @@ +"""Alembic environment for the Information Radiator.""" + +import os +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import create_engine + +from radiator.models import Base + +config = context.config +if config.config_file_name is not None: + # leave the app's own loggers (eg "radiator") working when migrations run + # in the same process, as in the tests + fileConfig(config.config_file_name, disable_existing_loggers=False) + + +def database_url() -> str: + url = config.attributes.get("database_url") or os.getenv("RADIATOR_DATABASE_URL") + if not url: + raise RuntimeError("RADIATOR_DATABASE_URL must be set") + return url + + +def run_migrations_offline(): + context.configure( + url=database_url(), + target_metadata=Base.metadata, + literal_binds=True, + compare_type=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + engine = create_engine(database_url()) + with engine.connect() as connection: + context.configure( + connection=connection, + target_metadata=Base.metadata, + compare_type=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/radiator/migrations/script.py.mako b/radiator/migrations/script.py.mako new file mode 100644 index 0000000..fe3b097 --- /dev/null +++ b/radiator/migrations/script.py.mako @@ -0,0 +1,25 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +${imports if imports else ""} + +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/radiator/migrations/versions/0001_initial_schema.py b/radiator/migrations/versions/0001_initial_schema.py new file mode 100644 index 0000000..4540f80 --- /dev/null +++ b/radiator/migrations/versions/0001_initial_schema.py @@ -0,0 +1,136 @@ +"""Initial schema + +Revision ID: 0001 +Revises: +Create Date: 2026-10-06 13:59:11.385508 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0001" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "runs", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("suite", sa.String(length=255), nullable=False), + sa.Column("env", sa.String(length=32), nullable=True), + sa.Column("target", sa.String(length=255), nullable=True), + sa.Column("target_url", sa.Text(), nullable=True), + sa.Column("query_type", sa.String(length=32), nullable=True), + sa.Column("harness_version", sa.String(length=64), nullable=True), + sa.Column("tests_source", sa.Text(), nullable=True), + sa.Column("origin", sa.String(length=32), nullable=False), + sa.Column("origin_ref", sa.String(length=255), nullable=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("ended_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("counts", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_runs_series", "runs", ["suite", "env", "started_at"], unique=False + ) + op.create_index("ix_runs_started_at", "runs", ["started_at"], unique=False) + op.create_index( + "uq_runs_origin_ref", + "runs", + ["origin", "origin_ref"], + unique=True, + postgresql_where="origin_ref IS NOT NULL", + ) + op.create_table( + "asset_results", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("run_id", sa.Uuid(), nullable=False), + sa.Column("test_case_id", sa.String(length=255), nullable=False), + sa.Column("asset_id", sa.String(length=255), nullable=False), + sa.Column("kind", sa.String(length=32), nullable=False), + sa.Column("name", sa.Text(), nullable=True), + sa.Column("expected_output", sa.String(length=64), nullable=True), + sa.Column("predicate", sa.String(length=255), nullable=True), + sa.Column("input_curie", sa.String(length=255), nullable=True), + sa.Column("output_curie", sa.String(length=255), nullable=True), + sa.Column("status", sa.String(length=32), nullable=False), + sa.Column("parent_pk", sa.String(length=255), nullable=True), + sa.Column("details", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.ForeignKeyConstraint(["run_id"], ["runs.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("run_id", "test_case_id", "asset_id"), + ) + op.create_index( + "ix_asset_results_asset", + "asset_results", + ["test_case_id", "asset_id"], + unique=False, + ) + op.create_table( + "performance_results", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("run_id", sa.Uuid(), nullable=False), + sa.Column("test_case_id", sa.String(length=255), nullable=False), + sa.Column("asset_id", sa.String(length=255), nullable=False), + sa.Column("host", sa.String(length=512), nullable=False), + sa.Column("helmsdeep_target", sa.String(length=64), nullable=True), + sa.Column("profile", sa.String(length=64), nullable=True), + sa.Column("status", sa.String(length=32), nullable=False), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("exit_code", sa.Integer(), nullable=True), + sa.Column("max_sustainable_concurrency", sa.Float(), nullable=True), + sa.Column("knee_unsupported", sa.Boolean(), nullable=True), + sa.Column("checkpoints_passed", sa.Boolean(), nullable=True), + sa.Column("summary", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.ForeignKeyConstraint(["run_id"], ["runs.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("run_id", "test_case_id", "asset_id", "host"), + ) + op.create_index( + "ix_performance_results_host", "performance_results", ["host"], unique=False + ) + op.create_table( + "agent_results", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("asset_result_id", sa.Integer(), nullable=False), + sa.Column("agent", sa.String(length=64), nullable=False), + sa.Column("status", sa.String(length=32), nullable=False), + sa.Column("message", sa.Text(), nullable=True), + sa.Column("http_status", sa.Integer(), nullable=True), + sa.Column("found", sa.Boolean(), nullable=True), + sa.Column("rank", sa.Integer(), nullable=True), + sa.Column("score", sa.Float(), nullable=True), + sa.Column("n_results", sa.Integer(), nullable=True), + sa.Column("response_time_s", sa.Float(), nullable=True), + sa.Column("pk", sa.String(length=255), nullable=True), + sa.Column("expected_nodes_found", sa.Text(), nullable=True), + sa.ForeignKeyConstraint( + ["asset_result_id"], ["asset_results.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("asset_result_id", "agent"), + ) + op.create_index("ix_agent_results_agent", "agent_results", ["agent"], unique=False) + + +def downgrade() -> None: + op.drop_index("ix_agent_results_agent", table_name="agent_results") + op.drop_table("agent_results") + op.drop_index("ix_performance_results_host", table_name="performance_results") + op.drop_table("performance_results") + op.drop_index("ix_asset_results_asset", table_name="asset_results") + op.drop_table("asset_results") + op.drop_index( + "uq_runs_origin_ref", + table_name="runs", + postgresql_where="origin_ref IS NOT NULL", + ) + op.drop_index("ix_runs_started_at", table_name="runs") + op.drop_index("ix_runs_series", table_name="runs") + op.drop_table("runs") diff --git a/radiator/migrations/versions/0002_record_query_parameters_on_runs.py b/radiator/migrations/versions/0002_record_query_parameters_on_runs.py new file mode 100644 index 0000000..4711879 --- /dev/null +++ b/radiator/migrations/versions/0002_record_query_parameters_on_runs.py @@ -0,0 +1,30 @@ +"""Record query parameters on runs + +Revision ID: 0002 +Revises: 0001 +Create Date: 2026-10-09 19:14:35.729463 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0002" +down_revision: Union[str, None] = "0001" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "runs", + sa.Column( + "query_parameters", postgresql.JSONB(astext_type=sa.Text()), nullable=True + ), + ) + + +def downgrade() -> None: + op.drop_column("runs", "query_parameters") diff --git a/radiator/migrations/versions/0003_number_runs.py b/radiator/migrations/versions/0003_number_runs.py new file mode 100644 index 0000000..46db926 --- /dev/null +++ b/radiator/migrations/versions/0003_number_runs.py @@ -0,0 +1,47 @@ +"""Number runs + +Revision ID: 0003 +Revises: 0002 +Create Date: 2026-10-10 + +Existing runs are numbered by start time; new ones get the next number as +they arrive. +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0003" +down_revision: Union[str, None] = "0002" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute("CREATE SEQUENCE run_number_seq") + op.add_column("runs", sa.Column("number", sa.Integer(), nullable=True)) + op.execute(""" + UPDATE runs SET number = numbered.n + FROM ( + SELECT id, row_number() OVER (ORDER BY started_at, id) AS n FROM runs + ) AS numbered + WHERE runs.id = numbered.id + """) + op.execute( + "SELECT setval('run_number_seq', COALESCE((SELECT max(number) FROM runs), 0) + 1, false)" + ) + op.alter_column( + "runs", + "number", + nullable=False, + server_default=sa.text("nextval('run_number_seq')"), + ) + op.create_unique_constraint("runs_number_key", "runs", ["number"]) + + +def downgrade() -> None: + op.drop_constraint("runs_number_key", "runs", type_="unique") + op.drop_column("runs", "number") + op.execute("DROP SEQUENCE run_number_seq") diff --git a/radiator/migrations/versions/0004_run_cycles.py b/radiator/migrations/versions/0004_run_cycles.py new file mode 100644 index 0000000..db6d412 --- /dev/null +++ b/radiator/migrations/versions/0004_run_cycles.py @@ -0,0 +1,28 @@ +"""Run cycles + +Revision ID: 0004 +Revises: 0003 +Create Date: 2026-10-10 + +Runs started by ``test-harness-cycle`` share a cycle id. +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0004" +down_revision: Union[str, None] = "0003" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("runs", sa.Column("cycle_id", sa.Uuid(), nullable=True)) + op.create_index("ix_runs_cycle_id", "runs", ["cycle_id"]) + + +def downgrade() -> None: + op.drop_index("ix_runs_cycle_id", table_name="runs") + op.drop_column("runs", "cycle_id") diff --git a/radiator/migrations/versions/0005_expected_assets.py b/radiator/migrations/versions/0005_expected_assets.py new file mode 100644 index 0000000..1af36f3 --- /dev/null +++ b/radiator/migrations/versions/0005_expected_assets.py @@ -0,0 +1,26 @@ +"""Expected assets + +Revision ID: 0005 +Revises: 0004 +Create Date: 2026-10-14 + +How many assets a run will test, for its progress while it's running. +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0005" +down_revision: Union[str, None] = "0004" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("runs", sa.Column("expected_assets", sa.Integer(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("runs", "expected_assets") diff --git a/radiator/migrations/versions/0006_trapi_version.py b/radiator/migrations/versions/0006_trapi_version.py new file mode 100644 index 0000000..a04e5d4 --- /dev/null +++ b/radiator/migrations/versions/0006_trapi_version.py @@ -0,0 +1,26 @@ +"""TRAPI version + +Revision ID: 0006 +Revises: 0005 +Create Date: 2026-10-15 + +The TRAPI version a run's queries were written in. +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0006" +down_revision: Union[str, None] = "0005" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("runs", sa.Column("trapi_version", sa.String(32), nullable=True)) + + +def downgrade() -> None: + op.drop_column("runs", "trapi_version") diff --git a/radiator/models.py b/radiator/models.py new file mode 100644 index 0000000..65e2991 --- /dev/null +++ b/radiator/models.py @@ -0,0 +1,183 @@ +"""Database tables. + +The shape follows ``radiator_schema``: a run has asset results, each with a row +per agent, plus any performance results. The unique constraints are the +upsert keys the ingest API relies on, so a retried or replayed upload updates +rows rather than duplicating them. +""" + +import uuid +from datetime import datetime, timezone +from typing import Any, Optional + +from sqlalchemy import ( + Boolean, + DateTime, + Float, + ForeignKey, + Index, + Integer, + Sequence, + String, + Text, + UniqueConstraint, + Uuid, +) +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + +# Postgres only: the ingest upserts use INSERT ... ON CONFLICT. +JsonType = JSONB + + +def utcnow(): + return datetime.now(timezone.utc) + + +class Base(DeclarativeBase): + pass + + +# Runs are numbered #1, #2, ... in the order they reach the radiator, for +# people to refer to them by. The database assigns the number on insert, so a +# re-uploaded run keeps its own. +RUN_NUMBER_SEQ = Sequence("run_number_seq") + + +class Run(Base): + __tablename__ = "runs" + + id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True) + number: Mapped[int] = mapped_column( + Integer, RUN_NUMBER_SEQ, server_default=RUN_NUMBER_SEQ.next_value(), unique=True + ) + suite: Mapped[str] = mapped_column(String(255)) + env: Mapped[Optional[str]] = mapped_column(String(32)) + target: Mapped[Optional[str]] = mapped_column(String(255)) + target_url: Mapped[Optional[str]] = mapped_column(Text) + query_type: Mapped[Optional[str]] = mapped_column(String(32)) + query_parameters: Mapped[Optional[dict[str, Any]]] = mapped_column(JsonType) + harness_version: Mapped[Optional[str]] = mapped_column(String(64)) + tests_source: Mapped[Optional[str]] = mapped_column(Text) + origin: Mapped[str] = mapped_column(String(32), default="harness") + origin_ref: Mapped[Optional[str]] = mapped_column(String(255)) + # the runs of one run cycle share this (see radiator_schema.RunCreate) + cycle_id: Mapped[Optional[uuid.UUID]] = mapped_column(Uuid, index=True) + # for the progress bar while the run is going (see RunCreate) + expected_assets: Mapped[Optional[int]] = mapped_column(Integer) + trapi_version: Mapped[Optional[str]] = mapped_column(String(32)) + started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + ended_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True)) + counts: Mapped[dict[str, Any]] = mapped_column(JsonType, default=dict) + # Also touched by every results upload: an open run that hasn't changed + # in a while is taken as one that died rather than one still going. + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utcnow, onupdate=utcnow + ) + + assets: Mapped[list["AssetResult"]] = relationship( + back_populates="run", cascade="all, delete-orphan", passive_deletes=True + ) + performance: Mapped[list["PerformanceResult"]] = relationship( + back_populates="run", cascade="all, delete-orphan", passive_deletes=True + ) + + __table_args__ = ( + # "the runs of this series", newest first: the runs list, and finding + # the previous comparable run for a diff + Index("ix_runs_series", "suite", "env", "started_at"), + Index("ix_runs_started_at", "started_at"), + # an import is re-runnable: one row per source run + Index( + "uq_runs_origin_ref", + "origin", + "origin_ref", + unique=True, + postgresql_where="origin_ref IS NOT NULL", + ), + ) + + +class AssetResult(Base): + __tablename__ = "asset_results" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + run_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("runs.id", ondelete="CASCADE")) + test_case_id: Mapped[str] = mapped_column(String(255)) + asset_id: Mapped[str] = mapped_column(String(255)) + kind: Mapped[str] = mapped_column(String(32)) + name: Mapped[Optional[str]] = mapped_column(Text) + expected_output: Mapped[Optional[str]] = mapped_column(String(64)) + predicate: Mapped[Optional[str]] = mapped_column(String(255)) + input_curie: Mapped[Optional[str]] = mapped_column(String(255)) + output_curie: Mapped[Optional[str]] = mapped_column(String(255)) + status: Mapped[str] = mapped_column(String(32)) + parent_pk: Mapped[Optional[str]] = mapped_column(String(255)) + details: Mapped[dict[str, Any]] = mapped_column(JsonType, default=dict) + + run: Mapped[Run] = relationship(back_populates="assets") + agents: Mapped[list["AgentResult"]] = relationship( + back_populates="asset", + cascade="all, delete-orphan", + passive_deletes=True, + order_by="AgentResult.id", + ) + + __table_args__ = ( + UniqueConstraint("run_id", "test_case_id", "asset_id"), + # an asset's history across runs + Index("ix_asset_results_asset", "test_case_id", "asset_id"), + ) + + +class AgentResult(Base): + __tablename__ = "agent_results" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + asset_result_id: Mapped[int] = mapped_column( + ForeignKey("asset_results.id", ondelete="CASCADE") + ) + agent: Mapped[str] = mapped_column(String(64)) + status: Mapped[str] = mapped_column(String(32)) + message: Mapped[Optional[str]] = mapped_column(Text) + http_status: Mapped[Optional[int]] = mapped_column(Integer) + found: Mapped[Optional[bool]] = mapped_column(Boolean) + rank: Mapped[Optional[int]] = mapped_column(Integer) + score: Mapped[Optional[float]] = mapped_column(Float) + n_results: Mapped[Optional[int]] = mapped_column(Integer) + response_time_s: Mapped[Optional[float]] = mapped_column(Float) + pk: Mapped[Optional[str]] = mapped_column(String(255)) + expected_nodes_found: Mapped[Optional[str]] = mapped_column(Text) + + asset: Mapped[AssetResult] = relationship(back_populates="agents") + + __table_args__ = ( + UniqueConstraint("asset_result_id", "agent"), + Index("ix_agent_results_agent", "agent"), + ) + + +class PerformanceResult(Base): + __tablename__ = "performance_results" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + run_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("runs.id", ondelete="CASCADE")) + test_case_id: Mapped[str] = mapped_column(String(255)) + asset_id: Mapped[str] = mapped_column(String(255)) + host: Mapped[str] = mapped_column(String(512)) + helmsdeep_target: Mapped[Optional[str]] = mapped_column(String(64)) + profile: Mapped[Optional[str]] = mapped_column(String(64)) + status: Mapped[str] = mapped_column(String(32)) + error: Mapped[Optional[str]] = mapped_column(Text) + exit_code: Mapped[Optional[int]] = mapped_column(Integer) + max_sustainable_concurrency: Mapped[Optional[float]] = mapped_column(Float) + knee_unsupported: Mapped[Optional[bool]] = mapped_column(Boolean) + checkpoints_passed: Mapped[Optional[bool]] = mapped_column(Boolean) + summary: Mapped[dict[str, Any]] = mapped_column(JsonType, default=dict) + + run: Mapped[Run] = relationship(back_populates="performance") + + __table_args__ = ( + UniqueConstraint("run_id", "test_case_id", "asset_id", "host"), + Index("ix_performance_results_host", "host"), + ) diff --git a/radiator/queries.py b/radiator/queries.py new file mode 100644 index 0000000..17fcb36 --- /dev/null +++ b/radiator/queries.py @@ -0,0 +1,732 @@ +"""Read queries behind the dashboard's views and the read API.""" + +import uuid +from collections import defaultdict +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from typing import Optional + +from sqlalchemy import func, select +from sqlalchemy.orm import Session, selectinload + +from radiator.models import AgentResult, AssetResult, PerformanceResult, Run + +STATUSES = ["PASSED", "FAILED", "NO_RESULTS", "ERROR", "SKIPPED"] +# Statuses that mean the agent answered and got it wrong, as opposed to the +# test not happening (SKIPPED). +NOT_PASSING = {"FAILED", "NO_RESULTS", "ERROR"} + +# Fixed display order, so the ARS leads and an agent's column (and color) never +# moves between runs. Agents not listed sort after these, alphabetically. +AGENT_ORDER = [ + "ars", + "aragorn", + "arax", + "biothings-explorer", + "improving-agent", + "unsecret-agent", + "cqs", + "shepherd-aragorn", + "shepherd-arax", + "shepherd-bte", +] + + +def agent_sort_key(agent: str): + try: + return (0, AGENT_ORDER.index(agent), agent) + except ValueError: + return (1, 0, agent) + + +def pass_rate(counts: dict) -> Optional[float]: + """Passed over everything that actually ran (skips excluded).""" + ran = sum(counts.get(status, 0) for status in STATUSES if status != "SKIPPED") + if ran == 0: + return None + return counts.get("PASSED", 0) / ran + + +# --- runs ------------------------------------------------------------------ + + +def list_runs( + session: Session, + suite: Optional[str] = None, + env: Optional[str] = None, + limit: int = 50, + offset: int = 0, +) -> list[Run]: + stmt = select(Run).order_by(Run.started_at.desc()).limit(limit).offset(offset) + if suite: + stmt = stmt.where(Run.suite == suite) + if env: + stmt = stmt.where(Run.env == env) + return list(session.scalars(stmt)) + + +def count_runs(session: Session, suite=None, env=None) -> int: + stmt = select(func.count()).select_from(Run) + if suite: + stmt = stmt.where(Run.suite == suite) + if env: + stmt = stmt.where(Run.env == env) + return session.scalar(stmt) + + +def filter_options(session: Session) -> dict: + return { + "suites": list( + session.scalars(select(Run.suite).distinct().order_by(Run.suite)) + ), + "envs": [ + env + for env in session.scalars(select(Run.env).distinct().order_by(Run.env)) + if env + ], + } + + +def status_counts(session: Session, run_ids: list[uuid.UUID]) -> dict: + """Overall (asset-level) status counts per run, from the stored results. + + Computed rather than read from ``Run.counts``: imported and unfinished runs + have no counts, and these always agree with what the run page shows. + """ + counts: dict = {run_id: {} for run_id in run_ids} + if not run_ids: + return counts + rows = session.execute( + select(AssetResult.run_id, AssetResult.status, func.count()) + .where(AssetResult.run_id.in_(run_ids)) + .group_by(AssetResult.run_id, AssetResult.status) + ) + for run_id, status, n in rows: + counts[run_id][status] = n + return counts + + +def performance_counts(session: Session, run_ids: list[uuid.UUID]) -> dict: + """Performance verdict counts per run, for runs with performance results.""" + counts: dict = defaultdict(dict) + if not run_ids: + return counts + rows = session.execute( + select(PerformanceResult.run_id, PerformanceResult.status, func.count()) + .where(PerformanceResult.run_id.in_(run_ids)) + .group_by(PerformanceResult.run_id, PerformanceResult.status) + ) + for run_id, status, n in rows: + counts[run_id][status] = n + return counts + + +def get_run(session: Session, run_id: uuid.UUID) -> Optional[Run]: + return session.scalar( + select(Run) + .where(Run.id == run_id) + .options( + selectinload(Run.assets).selectinload(AssetResult.agents), + selectinload(Run.performance), + ) + ) + + +def previous_run(session: Session, run: Run) -> Optional[Run]: + """The run before this one in the same series. + + A series is the same suite, env, target override, and query type: a run + against a locally running ARA, or of only the MVP1 queries, isn't + comparable with a full run against the deployed services. + """ + return session.scalar( + select(Run) + .where( + *_same_series(run), + Run.started_at < run.started_at, + # a crashed or still-uploading run isn't a fair baseline + Run.ended_at.is_not(None), + ) + .order_by(Run.started_at.desc()) + .limit(1) + ) + + +def _same_series(run: Run) -> list: + return [ + Run.suite == run.suite, + Run.env.is_not_distinct_from(run.env), + Run.target.is_not_distinct_from(run.target), + Run.query_type.is_not_distinct_from(run.query_type), + ] + + +def series_history( + session: Session, run: Run, limit: int = 30 +) -> list[tuple[Run, dict[str, dict[str, int]]]]: + """The finished runs of this run's series up to and including it, oldest + first, each with its per-agent status counts.""" + earlier = list( + session.scalars( + select(Run) + .where( + *_same_series(run), + Run.started_at < run.started_at, + Run.ended_at.is_not(None), + ) + .order_by(Run.started_at.desc()) + .limit(max(limit - 1, 0)) + ) + ) + runs = list(reversed(earlier)) + [run] + rows = session.execute( + select(AssetResult.run_id, AgentResult.agent, AgentResult.status, func.count()) + .join(AgentResult, AgentResult.asset_result_id == AssetResult.id) + .where(AssetResult.run_id.in_([r.id for r in runs])) + .group_by(AssetResult.run_id, AgentResult.agent, AgentResult.status) + ) + counts: dict = {r.id: defaultdict(dict) for r in runs} + for run_id, agent, status, n in rows: + counts[run_id][agent][status] = n + return [(r, dict(counts[r.id])) for r in runs] + + +def _performance_summary(session: Session, run: Run) -> list[dict]: + """Each service's top concurrency in this run against its previous run.""" + run = get_run(session, run.id) + out = [] + for perf, history in performance_series(session, run, limit=2): + previous = history[-2][1] if len(history) > 1 else None + out.append( + { + "host": perf.host, + "helmsdeep_target": perf.helmsdeep_target, + "profile": perf.profile, + "status": perf.status, + "error": perf.error, + "max_sustainable_concurrency": perf.max_sustainable_concurrency, + "knee_unsupported": perf.knee_unsupported, + "checkpoints_passed": perf.checkpoints_passed, + "previous_max_sustainable_concurrency": ( + previous.max_sustainable_concurrency if previous else None + ), + } + ) + return out + + +def trapi_minor(version: Optional[str]) -> Optional[str]: + """``"2.0.0"`` -> ``"2.0"``; None for an unknown version.""" + if not version: + return None + parts = version.split(".") + return ".".join(parts[:2]) if len(parts) >= 2 else version + + +def trapi_change(previous: Optional[Run], run: Run) -> Optional[tuple[str, str]]: + """(from, to) when the run's TRAPI version differs from the previous run's + (by major.minor); None when it's the same, or either isn't known.""" + if previous is None: + return None + before, after = trapi_minor(previous.trapi_version), trapi_minor(run.trapi_version) + if before and after and before != after: + return before, after + return None + + +def run_summary(session: Session, run: Run) -> dict: + """The headline of a run against the previous one in its series, for + notifications (eg the harness's Slack report).""" + overall = status_counts(session, [run.id])[run.id] + previous = previous_run(session, run) + summary = { + "run_id": str(run.id), + "performance": _performance_summary(session, run), + "pass_rate": pass_rate(overall), + "counts": overall, + "previous_run_id": None, + "previous_pass_rate": None, + "regressions": None, + "fixed": None, + "trapi_version": run.trapi_version, + "previous_trapi_version": None, + } + if previous is not None: + diff = diff_runs(get_run(session, previous.id), get_run(session, run.id)) + summary.update( + previous_trapi_version=previous.trapi_version, + previous_run_id=str(previous.id), + previous_pass_rate=pass_rate( + status_counts(session, [previous.id])[previous.id] + ), + regressions=diff.count("regression"), + fixed=diff.count("fixed"), + ) + return summary + + +def run_agents(run: Run) -> list[str]: + agents = {agent.agent for asset in run.assets for agent in asset.agents} + return sorted(agents, key=agent_sort_key) + + +# The acceptance tests' expected outputs, best-ranked answer first. +EXPECTED_OUTPUTS = ["TopAnswer", "Acceptable", "BadButForgivable", "NeverShow"] + + +def expected_output_sort_key(expected: Optional[str]): + if expected in EXPECTED_OUTPUTS: + return (0, EXPECTED_OUTPUTS.index(expected), "") + # anything else after those, alphabetically; no expected output last + return (1 if expected else 2, 0, expected or "") + + +def agent_expected_counts( + run: Run, +) -> dict[str, list[tuple[Optional[str], dict[str, int]]]]: + """Each agent's status counts, split by the assets' expected output. + + ``{agent: [(expected_output, {status: n}), ...]}``, expected outputs in + the order of EXPECTED_OUTPUTS. An asset without one is under None. + """ + counts: dict = defaultdict(lambda: defaultdict(lambda: defaultdict(int))) + for asset in run.assets: + for agent in asset.agents: + counts[agent.agent][asset.expected_output][agent.status] += 1 + return { + agent: sorted( + ((expected, dict(c)) for expected, c in by_expected.items()), + key=lambda item: expected_output_sort_key(item[0]), + ) + for agent, by_expected in counts.items() + } + + +def agent_counts(run: Run) -> dict[str, dict[str, int]]: + counts: dict = defaultdict(lambda: defaultdict(int)) + for asset in run.assets: + for agent in asset.agents: + counts[agent.agent][agent.status] += 1 + return counts + + +# --- diff ------------------------------------------------------------------ + +DIFF_KINDS = [ + ("regression", "Regressions", "Passed last run, not passing now"), + ("fixed", "Fixed", "Not passing last run, passing now"), + ("changed", "Other status changes", "Changed between non-passing statuses"), + ("skipped", "Skips", "Started or stopped being skipped"), + ("rank", "Rank moves", "Same status, expected answer moved 5 or more places"), + ("added", "New assets", "Not in the previous run"), + ("removed", "Dropped assets", "In the previous run, not in this one"), +] + + +@dataclass +class Change: + kind: str + asset: AssetResult + agent: Optional[str] = None + before: Optional[AgentResult] = None + after: Optional[AgentResult] = None + + @property + def rank_delta(self) -> Optional[int]: + """Positive when the expected answer moved up (a smaller rank).""" + if self.before is None or self.after is None: + return None + if self.before.rank is None or self.after.rank is None: + return None + return self.before.rank - self.after.rank + + +@dataclass +class Diff: + base: Optional[Run] + changes: dict[str, list[Change]] = field(default_factory=dict) + + def count(self, kind: str) -> int: + return len(self.changes.get(kind, [])) + + +# Smaller rank moves are mostly scoring noise between runs. +RANK_MOVE_THRESHOLD = 5 + + +def _classify(before: AgentResult, after: AgentResult) -> Optional[str]: + if before.status != after.status: + if "SKIPPED" in (before.status, after.status): + return "skipped" + if before.status == "PASSED": + return "regression" + if after.status == "PASSED": + return "fixed" + return "changed" + if ( + before.rank is not None + and after.rank is not None + and abs(before.rank - after.rank) >= RANK_MOVE_THRESHOLD + ): + return "rank" + return None + + +def diff_runs(base: Optional[Run], run: Run) -> Diff: + diff = Diff(base=base, changes={kind: [] for kind, _, _ in DIFF_KINDS}) + if base is None: + return diff + before_assets = {(a.test_case_id, a.asset_id): a for a in base.assets} + after_assets = {(a.test_case_id, a.asset_id): a for a in run.assets} + + for key, asset in after_assets.items(): + previous = before_assets.get(key) + if previous is None: + diff.changes["added"].append(Change("added", asset)) + continue + before_agents = {a.agent: a for a in previous.agents} + for after in asset.agents: + before = before_agents.get(after.agent) + if before is None: + continue + kind = _classify(before, after) + if kind: + diff.changes[kind].append( + Change(kind, asset, after.agent, before, after) + ) + for key, asset in before_assets.items(): + if key not in after_assets: + diff.changes["removed"].append(Change("removed", asset)) + + for changes in diff.changes.values(): + changes.sort( + key=lambda c: ( + agent_sort_key(c.agent or ""), + -abs(c.rank_delta or 0), + c.asset.name or "", + ) + ) + return diff + + +# --- asset history ----------------------------------------------------------- + +# How the asset page names "runs with no environment" (eg imported ones). +NO_ENV = "-" + + +@dataclass +class AssetHistory: + test_case_id: str + asset_id: str + latest: Optional[AssetResult] + # oldest first + points: list[tuple[Run, AssetResult]] + agents: list[str] + + def agent_result(self, asset: AssetResult, agent: str) -> Optional[AgentResult]: + for result in asset.agents: + if result.agent == agent: + return result + return None + + +def asset_history( + session: Session, + test_case_id: str, + asset_id: str, + env: Optional[str] = None, + limit: int = 60, +) -> AssetHistory: + stmt = ( + select(Run, AssetResult) + .join(AssetResult, AssetResult.run_id == Run.id) + .where( + AssetResult.test_case_id == test_case_id, + AssetResult.asset_id == asset_id, + ) + .options(selectinload(AssetResult.agents)) + .order_by(Run.started_at.desc()) + .limit(limit) + ) + if env == NO_ENV: + stmt = stmt.where(Run.env.is_(None)) + elif env: + stmt = stmt.where(Run.env == env) + points = list(reversed(session.execute(stmt).all())) + agents = sorted( + {agent.agent for _, asset in points for agent in asset.agents}, + key=agent_sort_key, + ) + return AssetHistory( + test_case_id=test_case_id, + asset_id=asset_id, + latest=points[-1][1] if points else None, + points=points, + agents=agents, + ) + + +def asset_envs(session: Session, test_case_id: str, asset_id: str) -> list[str]: + """The environments an asset has history in, most history first. + + Runs with no environment are listed as ``NO_ENV``. + """ + rows = session.execute( + select(Run.env, func.count()) + .join(AssetResult, AssetResult.run_id == Run.id) + .where( + AssetResult.test_case_id == test_case_id, + AssetResult.asset_id == asset_id, + ) + .group_by(Run.env) + .order_by(func.count().desc(), Run.env) + ) + return [env or NO_ENV for env, _ in rows] + + +# --- trends ---------------------------------------------------------------- + + +@dataclass +class TrendPoint: + run: Run + # agent -> expected_output -> status -> count + counts: dict + + +def agent_trends( + session: Session, suite: str, env: Optional[str], days: int = 90 +) -> list[TrendPoint]: + """Per-run, per-agent status counts by expected output, oldest first. + + Only full runs of the suite: target overrides and single-query-type runs + measure something else and would make the line jump. + """ + since = datetime.now(timezone.utc) - timedelta(days=days) + run_filter = [ + Run.suite == suite, + Run.started_at >= since, + Run.target.is_(None), + Run.query_type.is_(None), + Run.ended_at.is_not(None), + ] + if env: + run_filter.append(Run.env == env) + runs = list( + session.scalars(select(Run).where(*run_filter).order_by(Run.started_at)) + ) + if not runs: + return [] + rows = session.execute( + select( + AssetResult.run_id, + AgentResult.agent, + AssetResult.expected_output, + AgentResult.status, + func.count(), + ) + .join(AgentResult, AgentResult.asset_result_id == AssetResult.id) + .where(AssetResult.run_id.in_([run.id for run in runs])) + .group_by( + AssetResult.run_id, + AgentResult.agent, + AssetResult.expected_output, + AgentResult.status, + ) + ) + by_run: dict = { + run.id: defaultdict(lambda: defaultdict(lambda: defaultdict(int))) + for run in runs + } + for run_id, agent, expected, status, n in rows: + by_run[run_id][agent][expected or "Unspecified"][status] += n + return [TrendPoint(run=run, counts=by_run[run.id]) for run in runs] + + +# --- performance ------------------------------------------------------------- + + +def performance_history( + session: Session, days: int = 180 +) -> dict[str, list[tuple[Run, PerformanceResult]]]: + """Performance results per host (and HelmsDeep target), oldest first.""" + since = datetime.now(timezone.utc) - timedelta(days=days) + rows = session.execute( + select(Run, PerformanceResult) + .join(PerformanceResult, PerformanceResult.run_id == Run.id) + .where(Run.started_at >= since) + .order_by(Run.started_at) + ) + series: dict = defaultdict(list) + for run, perf in rows: + series[perf_series_name(perf, run.env)].append((run, perf)) + return dict(sorted(series.items())) + + +def perf_series_name(perf: PerformanceResult, env: Optional[str] = None) -> str: + """A service's performance series, by name. + + A series is one host under one HelmsDeep run type and profile, in one + environment: a mixed-profile run and a default one load the service + differently, so their numbers aren't comparable. + """ + detail = ", ".join( + part for part in (perf.helmsdeep_target, perf.profile or "default", env) if part + ) + return f"{perf.host} ({detail})" + + +def performance_series( + session: Session, run: Run, limit: int = 30 +) -> list[tuple[PerformanceResult, list[tuple[Run, PerformanceResult]]]]: + """For each performance result in this run, its series' history up to and + including it, oldest first, from finished runs.""" + out = [] + for perf in sorted(run.performance, key=lambda p: p.host): + earlier = session.execute( + select(Run, PerformanceResult) + .join(PerformanceResult, PerformanceResult.run_id == Run.id) + .where( + PerformanceResult.host == perf.host, + PerformanceResult.helmsdeep_target.is_not_distinct_from( + perf.helmsdeep_target + ), + PerformanceResult.profile.is_not_distinct_from(perf.profile), + Run.env.is_not_distinct_from(run.env), + Run.started_at < run.started_at, + Run.ended_at.is_not(None), + ) + .order_by(Run.started_at.desc()) + .limit(max(limit - 1, 0)) + ).all() + out.append((perf, [tuple(row) for row in reversed(earlier)] + [(run, perf)])) + return out + + +# --- live updates --------------------------------------------------------------- + +RUNNING, STALLED, FINISHED = "running", "stalled", "finished" +# An open run that hasn't uploaded anything for this long is taken to have +# died (a crash, a killed pod) rather than to be still going. HelmsDeep's +# ~70 minutes without an upload is the longest quiet stretch a live run has. +STALE_AFTER = timedelta(hours=2) + + +def run_state(run: Run, now: Optional[datetime] = None) -> str: + if run.ended_at is not None: + return FINISHED + now = now or datetime.now(timezone.utc) + if run.updated_at is not None and now - run.updated_at > STALE_AFTER: + return STALLED + return RUNNING + + +def run_progress(session: Session, run: Run) -> dict: + """How far a run has got, cheaply: for the run page's live updates.""" + counts = status_counts(session, [run.id])[run.id] + performance = session.scalar( + select(func.count()) + .select_from(PerformanceResult) + .where(PerformanceResult.run_id == run.id) + ) + done = sum(counts.values()) + performance + state = run_state(run) + return { + "run_id": str(run.id), + "state": state, + "running": state == RUNNING, + "done_assets": done, + "expected_assets": run.expected_assets, + "counts": counts, + "pass_rate": pass_rate(counts), + "updated_at": run.updated_at.isoformat() if run.updated_at else None, + # changes whenever anything on the run page would + "version": f"{run.updated_at.isoformat() if run.updated_at else ''}" + f"|{run.ended_at.isoformat() if run.ended_at else ''}|{done}", + } + + +def live_status(session: Session) -> dict: + """What's running now, for the runs list's and the weekly page's live + updates. ``version`` changes when a run starts, uploads, or finishes.""" + since = datetime.now(timezone.utc) - STALE_AFTER + running = list( + session.execute( + select(Run.id, Run.updated_at) + .where(Run.ended_at.is_(None), Run.updated_at >= since) + .order_by(Run.started_at) + ) + ) + latest = session.execute( + select(Run.id, Run.ended_at).order_by(Run.started_at.desc()).limit(1) + ).first() + finished = session.scalar(select(func.max(Run.ended_at))) + parts = [str(latest.id) if latest else "", str(finished or "")] + parts += [f"{run_id}@{updated.isoformat()}" for run_id, updated in running] + return { + "running": [str(run_id) for run_id, _ in running], + "version": "|".join(parts), + } + + +# --- JSON for the read API --------------------------------------------------- + + +def run_json(run: Run, with_results: bool = False) -> dict: + data = { + "run_id": str(run.id), + "number": run.number, + "suite": run.suite, + "env": run.env, + "target": run.target, + "query_type": run.query_type, + "query_parameters": run.query_parameters, + "trapi_version": run.trapi_version, + "harness_version": run.harness_version, + "origin": run.origin, + "started_at": run.started_at.isoformat(), + "ended_at": run.ended_at.isoformat() if run.ended_at else None, + "counts": run.counts, + } + if with_results: + data["results"] = [ + { + "test_case_id": asset.test_case_id, + "asset_id": asset.asset_id, + "name": asset.name, + "kind": asset.kind, + "expected_output": asset.expected_output, + "input_curie": asset.input_curie, + "output_curie": asset.output_curie, + "status": asset.status, + "agents": [ + { + "agent": agent.agent, + "status": agent.status, + "found": agent.found, + "rank": agent.rank, + "score": agent.score, + "n_results": agent.n_results, + "http_status": agent.http_status, + "response_time_s": agent.response_time_s, + "pk": agent.pk, + } + for agent in asset.agents + ], + } + for asset in run.assets + ] + data["performance"] = [ + { + "host": perf.host, + "helmsdeep_target": perf.helmsdeep_target, + "profile": perf.profile, + "status": perf.status, + "max_sustainable_concurrency": perf.max_sustainable_concurrency, + "checkpoints_passed": perf.checkpoints_passed, + "error": perf.error, + } + for perf in run.performance + ] + return data diff --git a/radiator/requirements-test.txt b/radiator/requirements-test.txt new file mode 100644 index 0000000..29d64a6 --- /dev/null +++ b/radiator/requirements-test.txt @@ -0,0 +1,4 @@ +-r requirements.txt +pytest==9.1.1 +# Starlette's TestClient +httpx2==2.13.1 diff --git a/radiator/requirements.txt b/radiator/requirements.txt new file mode 100644 index 0000000..fe0fc7a --- /dev/null +++ b/radiator/requirements.txt @@ -0,0 +1,16 @@ +# The Information Radiator's own dependencies. Kept apart from the harness's +# requirements: the two ship as separate images and pin independently. The +# shared contract, radiator_schema, needs only pydantic 2 and works with both. +fastapi==0.142.2 +uvicorn[standard]==0.54.0 +sqlalchemy==2.1.3 +psycopg[binary]==3.3.6 +alembic==1.20.0 +jinja2==3.1.6 +itsdangerous==2.2.0 +python-multipart==0.0.32 +pydantic==2.13.5 +# the PNGs the harness posts to Slack: history, performance, board, digest +matplotlib==3.11.2 +# for the Zebrunner import +httpx==0.28.1 diff --git a/radiator/static/charts.js b/radiator/static/charts.js new file mode 100644 index 0000000..55c4593 --- /dev/null +++ b/radiator/static/charts.js @@ -0,0 +1,100 @@ +// Hover layer for the server-rendered charts and status cells. Tooltips only +// enhance: every value is also in the page's tables. All text is inserted with +// textContent, since names and messages come from uploaded data. +(function () { + const tip = document.createElement("div"); + tip.className = "tooltip"; + document.body.appendChild(tip); + + function place(event) { + const pad = 14; + let x = event.clientX + pad; + let y = event.clientY + pad; + const rect = tip.getBoundingClientRect(); + if (x + rect.width > window.innerWidth - 8) x = event.clientX - rect.width - pad; + if (y + rect.height > window.innerHeight - 8) y = event.clientY - rect.height - pad; + tip.style.left = x + "px"; + tip.style.top = y + "px"; + } + + function show(title, rows, event) { + tip.replaceChildren(); + const head = document.createElement("div"); + head.className = "tt-title"; + head.textContent = title; + tip.appendChild(head); + for (const row of rows) { + const line = document.createElement("div"); + line.className = "tt-row"; + if (row.color) { + const key = document.createElement("span"); + key.className = "tt-key"; + key.style.background = row.color; + line.appendChild(key); + } + const value = document.createElement("span"); + value.className = "tt-value"; + value.textContent = row.value; + const name = document.createElement("span"); + name.className = "tt-name"; + name.textContent = row.name; + line.append(value, name); + tip.appendChild(line); + } + tip.style.display = "block"; + place(event); + } + + function hide() { + tip.style.display = "none"; + } + + // Line charts: a crosshair that snaps to the nearest run, listing every series. + for (const wrap of document.querySelectorAll(".chart-wrap[data-hover]")) { + const data = JSON.parse(wrap.dataset.hover); + const svg = wrap.querySelector("svg"); + const cross = svg.querySelector(".crosshair"); + svg.addEventListener("pointermove", (event) => { + const box = svg.getBoundingClientRect(); + const x = ((event.clientX - box.left) / box.width) * data.width; + let best = 0; + for (let i = 1; i < data.xs.length; i++) { + if (Math.abs(data.xs[i] - x) < Math.abs(data.xs[best] - x)) best = i; + } + cross.setAttribute("x1", data.xs[best]); + cross.setAttribute("x2", data.xs[best]); + cross.setAttribute("visibility", "visible"); + const rows = data.series + .filter((s) => s.values[best] !== null) + .map((s) => ({ color: s.color, value: s.values[best], name: s.name })); + if (!rows.length) rows.push({ value: "no data", name: "" }); + show(data.titles[best], rows, event); + }); + svg.addEventListener("pointerleave", () => { + cross.setAttribute("visibility", "hidden"); + hide(); + }); + } + + // Cells and bar segments: each carries its own tooltip. + function cellTip(event) { + const el = event.target.closest("[data-tip]"); + if (!el) return; + const lines = el.dataset.tip.split("\n"); + show(lines[0], lines.slice(1).map((l) => ({ value: l, name: "" })), event); + } + document.addEventListener("pointerover", cellTip); + document.addEventListener("pointermove", (event) => { + if (event.target.closest("[data-tip]")) place(event); + }); + document.addEventListener("pointerout", (event) => { + if (event.target.closest("[data-tip]")) hide(); + }); + document.addEventListener("focusin", (event) => { + const el = event.target.closest("[data-tip]"); + if (!el) return; + const box = el.getBoundingClientRect(); + cellTip({ target: el, clientX: box.right, clientY: box.bottom }); + }); + document.addEventListener("focusout", hide); +})(); diff --git a/radiator/static/live.js b/radiator/static/live.js new file mode 100644 index 0000000..6ab3e94 --- /dev/null +++ b/radiator/static/live.js @@ -0,0 +1,90 @@ +// Live updates. A page that can change while it's open puts on
+// data-live="" JSON with a "version" (and "running") +// data-live-interval="10" seconds between checks +// data-live-version="..." the version the page was rendered at +// data-live-until-done stop once "running" is false (a run's page) +// and marks the parts that change with data-live-region and an id. When the +// version changes, the page is fetched again and those parts swapped in, so +// filters, pagination and the rest of the page stay as they are. Without +// JavaScript, the page is what it was: reload to see more. +(function () { + const main = document.querySelector("main[data-live]"); + if (!main) return; + const checkUrl = main.dataset.live; + const every = (Number(main.dataset.liveInterval) || 15) * 1000; + const untilDone = "liveUntilDone" in main.dataset; + const note = document.querySelector("[data-live-indicator]"); + let version = main.dataset.liveVersion || null; + let failures = 0; + let timer = null; + let stopped = false; + + function say(text) { + if (!note) return; + note.hidden = false; + note.textContent = text; + } + + function clock() { + return new Date().toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", second: "2-digit" }); + } + + async function refresh() { + const res = await fetch(location.href, { cache: "no-store", credentials: "same-origin" }); + if (!res.ok) throw new Error(`page: ${res.status}`); + const doc = new DOMParser().parseFromString(await res.text(), "text/html"); + for (const region of document.querySelectorAll("[data-live-region][id]")) { + const fresh = doc.getElementById(region.id); + if (fresh) region.replaceWith(document.importNode(fresh, true)); + } + } + + async function tick() { + timer = null; + if (stopped) return; + if (document.hidden) return; // picked up again when the tab is shown + try { + const res = await fetch(checkUrl, { + cache: "no-store", + credentials: "same-origin", + headers: { Accept: "application/json" }, + }); + if (res.status === 401 || res.status === 403) { + stopped = true; + say("Live updates stopped: log in again to resume."); + return; + } + if (!res.ok) throw new Error(`check: ${res.status}`); + const data = await res.json(); + // the first check only learns the version, unless the page gave one + if (version !== null && data.version !== version) await refresh(); + version = data.version; + failures = 0; + if (untilDone && !data.running) { + stopped = true; + say(`Finished · updated ${clock()}`); + return; + } + say(`Live · updated ${clock()}`); + } catch (err) { + failures += 1; + say(`Live updates paused (${failures} failed ${failures === 1 ? "try" : "tries"}); retrying…`); + } + schedule(); + } + + function schedule() { + if (stopped || timer !== null) return; + // back off while the radiator can't be reached + timer = setTimeout(tick, every * Math.min(2 ** failures, 8)); + } + + document.addEventListener("visibilitychange", () => { + if (document.hidden || stopped) return; + clearTimeout(timer); + timer = null; + tick(); + }); + say("Live"); + schedule(); +})(); diff --git a/radiator/static/style.css b/radiator/static/style.css new file mode 100644 index 0000000..041f212 --- /dev/null +++ b/radiator/static/style.css @@ -0,0 +1,435 @@ +/* Information Radiator. Colors are tokens: light by default, dark from the OS + setting. Status colors are fixed and always paired with an icon and label; + agent colors are fixed per agent so an agent looks the same everywhere. */ + +:root { + color-scheme: light; + --page: #f9f9f7; + --surface: #fcfcfb; + --surface-2: #f3f2ef; + --ink: #0b0b0b; + --ink-2: #52514e; + --ink-muted: #6f6d68; + --grid: #e1e0d9; + --axis: #c3c2b7; + --border: rgba(11, 11, 11, 0.1); + --link: #1c5cab; + --accent: #2a78d6; + --up: #006300; + --down: #b12f2f; + + --status-passed: #0ca30c; + --status-no-results: #fab219; + --status-error: #ec835a; + --status-failed: #d03b3b; + --status-skipped: #d6d5ce; + --grid-base: #898781; + --grid-excl: #b5b3ab; + /* verdicts: status colors, always shown with their word */ + --v-regressed: #d03b3b; + --v-mixed: #eda100; + --v-improved: #0ca30c; + --v-steady: #898781; + --v-first: #2a78d6; + --v-nodata: #52514e; + + --agent-1: #2a78d6; + --agent-2: #eb6834; + --agent-3: #1baf7a; + --agent-4: #eda100; + --agent-5: #e87ba4; + --agent-6: #008300; + --agent-7: #4a3aa7; + --agent-8: #e34948; +} + +@media (prefers-color-scheme: dark) { + :root { + color-scheme: dark; + --page: #0d0d0d; + --surface: #1a1a19; + --surface-2: #232321; + --ink: #ffffff; + --ink-2: #c3c2b7; + --ink-muted: #9a9890; + --grid: #2c2c2a; + --axis: #383835; + --border: rgba(255, 255, 255, 0.1); + --link: #86b6ef; + --accent: #3987e5; + --up: #0ca30c; + --down: #e66767; + --status-skipped: #4a4a46; + --grid-base: #9a9890; + --grid-excl: #5c5b57; + --v-regressed: #e66767; + --v-mixed: #c98500; + --v-improved: #0ca30c; + --v-steady: #9a9890; + --v-first: #3987e5; + --v-nodata: #c3c2b7; + + --agent-1: #3987e5; + --agent-2: #d95926; + --agent-3: #199e70; + --agent-4: #c98500; + --agent-5: #d55181; + --agent-6: #008300; + --agent-7: #9085e9; + --agent-8: #e66767; + } +} + +* { box-sizing: border-box; } + +body { + margin: 0; + background: var(--page); + color: var(--ink); + font: 14px/1.45 system-ui, -apple-system, "Segoe UI", sans-serif; +} + +a { color: var(--link); text-decoration: none; } +a:hover { text-decoration: underline; } + +code, .mono { + font-family: ui-monospace, "SF Mono", Menlo, Consolas, monospace; + font-size: 12px; +} + +/* --- frame ------------------------------------------------------------- */ + +.topbar { + display: flex; + align-items: center; + gap: 24px; + padding: 0 24px; + height: 52px; + background: var(--surface); + border-bottom: 1px solid var(--border); +} +.brand { + font-weight: 650; + color: var(--ink); + display: flex; + align-items: center; + gap: 8px; +} +.brand:hover { text-decoration: none; } +.brand-mark { + width: 18px; height: 18px; border-radius: 5px; + background: conic-gradient(var(--status-passed) 0 60%, var(--status-failed) 0 80%, var(--status-no-results) 0); +} +.nav { display: flex; gap: 4px; flex: 1; min-width: 0; overflow-x: auto; } +.nav a { + white-space: nowrap; + color: var(--ink-2); + padding: 6px 10px; + border-radius: 6px; +} +.nav a:hover { background: var(--surface-2); text-decoration: none; } +.nav a.active { color: var(--ink); background: var(--surface-2); font-weight: 550; } +.brand { white-space: nowrap; } +.logout button { white-space: nowrap; + background: none; border: 0; color: var(--ink-2); cursor: pointer; font: inherit; +} + +main { max-width: 1280px; margin: 0 auto; padding: 24px; } + +.page-head { margin-bottom: 20px; } +.page-head h1 { font-size: 22px; font-weight: 650; margin: 0 0 4px; } +.crumbs { color: var(--ink-muted); font-size: 13px; margin-bottom: 6px; } +.meta { color: var(--ink-2); display: flex; flex-wrap: wrap; gap: 6px 16px; } + +.card { + background: var(--surface); + border: 1px solid var(--border); + border-radius: 10px; + padding: 16px 20px; + margin-bottom: 20px; +} +.card h2 { font-size: 15px; font-weight: 650; margin: 0 0 2px; } +.card .sub { color: var(--ink-muted); font-size: 13px; margin: 0 0 12px; } +.card-head { display: flex; justify-content: space-between; align-items: baseline; gap: 16px; } + +.empty { color: var(--ink-muted); padding: 24px 0; text-align: center; } + +/* --- filters: one row above what they scope ------------------------------ */ + +.filters { + display: flex; flex-wrap: wrap; gap: 8px; align-items: end; margin-bottom: 16px; +} +.filters label { display: flex; flex-direction: column; font-size: 12px; color: var(--ink-muted); gap: 2px; } +.filters select, .filters input { + font: inherit; color: var(--ink); background: var(--surface); + border: 1px solid var(--axis); border-radius: 6px; padding: 5px 8px; min-width: 120px; +} +.filters button, .button { + font: inherit; border: 1px solid var(--axis); background: var(--surface); + color: var(--ink); border-radius: 6px; padding: 5px 12px; cursor: pointer; +} +.filters .clear { align-self: center; font-size: 13px; } + +/* --- stat tiles ---------------------------------------------------------- */ + +.tiles { display: grid; grid-template-columns: repeat(auto-fit, minmax(170px, 1fr)); gap: 12px; margin-bottom: 20px; } +.tile { + background: var(--surface); border: 1px solid var(--border); border-radius: 10px; padding: 14px 16px; +} +.tile .label { color: var(--ink-2); font-size: 13px; } +.tile .value { font-size: 26px; font-weight: 600; margin-top: 2px; } +.tile.hero .value { font-size: 48px; line-height: 1.1; } +.tile .delta { font-size: 13px; color: var(--ink-muted); } +.delta.up { color: var(--up); } +.delta.down { color: var(--down); } + +/* --- tables -------------------------------------------------------------- */ + +table.data { width: 100%; border-collapse: collapse; } +table.data th { + text-align: left; font-weight: 550; color: var(--ink-2); font-size: 12px; + padding: 6px 8px; border-bottom: 1px solid var(--axis); white-space: nowrap; + position: sticky; top: 0; background: var(--surface); +} +table.data td { padding: 6px 8px; border-bottom: 1px solid var(--grid); vertical-align: middle; } +table.data td.num, table.data th.num { text-align: right; font-variant-numeric: tabular-nums; } +table.data tr:hover td { background: var(--surface-2); } +/* rows grouped under a heading cell (eg an agent), groups set apart */ +table.grouped tbody + tbody tr:first-child > * { border-top: 1px solid var(--axis); } +table.grouped .group-head { + text-align: left; font-weight: 550; vertical-align: top; padding-top: 8px; + border-bottom: 1px solid var(--grid); background: var(--surface); + font-size: 14px; color: var(--ink); position: static; +} +.scroll { overflow-x: auto; } +.muted { color: var(--ink-muted); } +.nowrap { white-space: nowrap; } + +.badge { + display: inline-block; font-size: 11px; padding: 1px 7px; border-radius: 999px; + border: 1px solid var(--axis); color: var(--ink-2); white-space: nowrap; +} + +/* --- status: color + icon + label, never color alone -------------------- */ + +.status { + display: inline-flex; align-items: center; gap: 5px; white-space: nowrap; font-size: 13px; +} +.status .icon { + display: inline-grid; place-items: center; width: 18px; height: 18px; border-radius: 4px; + font-size: 11px; font-weight: 700; color: #0b0b0b; +} +.st-PASSED .icon, .cell.st-PASSED { background: var(--status-passed); } +/* In the matrix and history most cells pass: a tint keeps the page calm, so + what isn't passing is what stands out. */ +.matrix .cell.st-PASSED, .strip .cell.st-PASSED { + background: color-mix(in srgb, var(--status-passed) 16%, var(--surface)); + color: var(--up); + box-shadow: inset 0 0 0 1px color-mix(in srgb, var(--status-passed) 35%, transparent); +} +.st-NO_RESULTS .icon, .cell.st-NO_RESULTS { background: var(--status-no-results); } +.st-ERROR .icon, .cell.st-ERROR { background: var(--status-error); } +.st-FAILED .icon, .cell.st-FAILED { background: var(--status-failed); color: #fff; } +.st-SKIPPED .icon, .cell.st-SKIPPED { background: var(--status-skipped); color: var(--ink-2); } + +.legend { display: flex; flex-wrap: wrap; gap: 6px 14px; font-size: 12px; color: var(--ink-2); margin: 8px 0 0; } +.legend .key { display: inline-flex; align-items: center; gap: 6px; } +.legend .line { width: 16px; height: 2px; border-radius: 1px; } + +/* stacked status bar: part-to-whole, 2px surface gaps between segments */ +.stackbar { display: flex; gap: 2px; height: 10px; min-width: 120px; } +.stackbar span { display: block; height: 100%; min-width: 2px; } +.stackbar span:first-child { border-radius: 4px 0 0 4px; } +.stackbar span:last-child { border-radius: 0 4px 4px 0; } +.stackbar span:only-child { border-radius: 4px; } +.seg-PASSED { background: var(--status-passed); } +.seg-NO_RESULTS { background: var(--status-no-results); } +.seg-ERROR { background: var(--status-error); } +.seg-FAILED { background: var(--status-failed); } +.seg-SKIPPED { background: var(--status-skipped); } + +/* run matrix: one cell per asset x agent */ +.matrix td.agent-cell { padding: 3px 4px; } +.cell { + display: flex; align-items: center; justify-content: center; gap: 4px; + min-width: 64px; height: 26px; border-radius: 5px; font-size: 12px; + font-weight: 600; color: #0b0b0b; font-variant-numeric: tabular-nums; +} +.cell:hover { text-decoration: none; outline: 2px solid var(--ink); outline-offset: 1px; } +.cell .rank { font-weight: 500; opacity: 0.85; } +th.agent-head { text-align: center; } +.agent-dot { display: inline-block; width: 8px; height: 8px; border-radius: 50%; margin-right: 5px; vertical-align: 1px; } + +/* history strip: rows are agents, columns are runs, oldest -> newest */ +.strip { display: grid; gap: 2px; align-items: center; } +.strip .row-label { font-size: 12px; color: var(--ink-2); padding-right: 10px; white-space: nowrap; } +.strip .cell { min-width: 0; height: 22px; font-size: 10px; border-radius: 3px; } +.strip .axis-label { font-size: 11px; color: var(--ink-muted); white-space: nowrap; } + +/* diff */ +.diff-summary { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 16px; } +.diff-summary a { + border: 1px solid var(--border); background: var(--surface); border-radius: 8px; + padding: 8px 12px; color: var(--ink); min-width: 120px; +} +.diff-summary a:hover { text-decoration: none; border-color: var(--axis); } +.diff-summary .n { font-size: 20px; font-weight: 600; display: block; } +.diff-summary .k { font-size: 12px; color: var(--ink-2); } +.arrow { color: var(--ink-muted); margin: 0 4px; } + +/* charts */ +.chart { width: 100%; height: auto; display: block; overflow: visible; } +.chart text { fill: var(--ink-muted); font-size: 11px; font-variant-numeric: tabular-nums; } +.chart .grid { stroke: var(--grid); stroke-width: 1; } +.chart .baseline { stroke: var(--axis); stroke-width: 1; } +.chart .series { fill: none; stroke-width: 2; stroke-linejoin: miter; stroke-linecap: square; } +.chart .dot { stroke: var(--surface); stroke-width: 2; } +.chart .crosshair { stroke: var(--ink-muted); stroke-width: 1; } +.chart-wrap { position: relative; } +.multiples { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(380px, 100%), 1fr)); gap: 12px 24px; } +.multiples.two { grid-template-columns: repeat(auto-fit, minmax(min(480px, 100%), 1fr)); } +.multiples h3 { font-size: 13px; font-weight: 600; margin: 4px 0; } + +details.table-view { margin-top: 8px; } +details.table-view summary { cursor: pointer; color: var(--ink-2); font-size: 12px; } +details.table-view[open] summary { margin-bottom: 8px; } + +/* tooltip (one for the page; filled with textContent by charts.js) */ +.tooltip { + position: fixed; pointer-events: none; z-index: 10; display: none; + background: var(--surface); color: var(--ink); border: 1px solid var(--border); + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.12); border-radius: 8px; padding: 8px 10px; + font-size: 12px; max-width: 320px; +} +.tooltip .tt-title { color: var(--ink-2); margin-bottom: 4px; } +.tooltip .tt-row { display: flex; align-items: center; gap: 8px; } +.tooltip .tt-key { width: 12px; height: 2px; border-radius: 1px; flex: none; } +.tooltip .tt-value { font-weight: 650; font-variant-numeric: tabular-nums; } +.tooltip .tt-name { color: var(--ink-2); } + +/* login */ +.login-page { min-height: 100vh; display: grid; place-items: center; padding: 16px; } +.login { + width: 100%; max-width: 360px; background: var(--surface); border: 1px solid var(--border); + border-radius: 12px; padding: 28px; +} +.login h1 { font-size: 20px; margin: 0 0 4px; } +.login p { color: var(--ink-2); margin: 0 0 20px; } +.login label { display: block; font-size: 13px; color: var(--ink-2); margin-bottom: 12px; } +.login input { + display: block; width: 100%; margin-top: 4px; font: inherit; color: var(--ink); + background: var(--page); border: 1px solid var(--axis); border-radius: 6px; padding: 8px 10px; +} +.login button { + width: 100%; margin-top: 8px; font: inherit; font-weight: 600; color: #fff; + background: var(--accent); border: 0; border-radius: 6px; padding: 9px; cursor: pointer; +} +.error { + background: color-mix(in srgb, var(--status-failed) 12%, transparent); + border: 1px solid color-mix(in srgb, var(--status-failed) 40%, transparent); + border-radius: 6px; padding: 8px 10px; margin-bottom: 14px; font-size: 13px; +} +.pager-head, .pager-foot { + display: flex; flex-wrap: wrap; align-items: center; gap: 8px 16px; margin: 4px 0 8px; +} +.pager-head .per-page { margin-left: auto; } +.pager-head .per-page label { font-size: 13px; color: var(--ink-2); display: flex; align-items: center; gap: 6px; } +.pager-head .per-page select { + font: inherit; color: var(--ink); background: var(--surface); + border: 1px solid var(--axis); border-radius: 6px; padding: 3px 6px; +} +.pager-foot { justify-content: space-between; margin-top: 8px; } +.pager-foot .sub { margin: 0; } +.pager-bar { display: flex; align-items: center; gap: 4px; font-size: 13px; } +.pager-bar a, .pager-bar span { padding: 4px 8px; border-radius: 6px; white-space: nowrap; } +.pager-bar a:hover { background: var(--surface-2); text-decoration: none; } +.pager-bar .disabled { color: var(--ink-muted); opacity: 0.6; } +.pager-bar .where { color: var(--ink-2); } + +.narrow-only { display: none; } +@media (max-width: 640px) { + .wide-only { display: none; } + .narrow-only { display: block; } + main { padding: 16px; } + .topbar { padding: 0 16px; gap: 8px; } + .brand-text { display: none; } + .card { padding: 14px; } + .card-head { flex-direction: column; gap: 4px; } + .tile.hero .value { font-size: 40px; } +} + +/* run grid: trend colors are the status palette, always with their label */ +.run-grid { width: 100%; min-width: 900px; height: auto; display: block; } +.run-grid .band { fill: var(--surface); } +.run-grid .band-alt { fill: var(--surface-2); } +.run-grid .lane-label { font-size: 17px; font-weight: 650; fill: var(--ink); } +.run-grid .lane-note { font-size: 12px; fill: var(--ink-muted); } +.run-grid .tick { stroke: var(--grid); stroke-width: 1; } +.run-grid .tick-label { font-size: 12px; fill: var(--ink-muted); } +.run-grid .link { stroke-width: 3.5; } +.run-grid .link.t-excl, .run-grid .link.t-nodata { stroke-dasharray: 4 4; stroke-width: 2.5; } +.run-grid .card-box { stroke-width: 2; } +.run-grid .t-excl .card-box, .run-grid .t-nodata .card-box { stroke-dasharray: 5 4; } +.run-grid .chip-label { font-size: 10.5px; font-weight: 700; fill: #0b0b0b; letter-spacing: 0.03em; } +.run-grid .t-down .chip-label { fill: #fff; } +.run-grid .number { font-size: 14px; font-weight: 700; fill: var(--ink); } +.run-grid .when { font-size: 12px; fill: var(--ink-2); font-variant-numeric: tabular-nums; } +.run-grid .rate { font-size: 13.5px; font-weight: 650; fill: var(--ink); } +.run-grid .counts { font-size: 11.5px; fill: var(--ink-2); font-variant-numeric: tabular-nums; } +.run-grid a:hover .card-box, .run-grid a:focus .card-box { stroke-width: 3.5; } +.run-grid .t-up .card-box { fill: color-mix(in srgb, var(--status-passed) 12%, var(--surface)); stroke: var(--status-passed); } +.run-grid .t-down .card-box { fill: color-mix(in srgb, var(--status-failed) 12%, var(--surface)); stroke: var(--status-failed); } +.run-grid .t-flat .card-box { fill: color-mix(in srgb, var(--status-no-results) 14%, var(--surface)); stroke: var(--status-no-results); } +.run-grid .t-base .card-box { fill: var(--surface); stroke: var(--grid-base); } +.run-grid .t-excl .card-box, .run-grid .t-nodata .card-box { fill: var(--surface-2); stroke: var(--grid-excl); } +.run-grid .t-up .chip, .run-grid .link.t-up, .grid-key.t-up { fill: var(--status-passed); stroke: var(--status-passed); background: var(--status-passed); } +.run-grid .t-down .chip, .run-grid .link.t-down, .grid-key.t-down { fill: var(--status-failed); stroke: var(--status-failed); background: var(--status-failed); } +.run-grid .t-flat .chip, .run-grid .link.t-flat, .grid-key.t-flat { fill: var(--status-no-results); stroke: var(--status-no-results); background: var(--status-no-results); } +.run-grid .t-base .chip, .run-grid .link.t-base, .grid-key.t-base { fill: var(--grid-base); stroke: var(--grid-base); background: var(--grid-base); } +.run-grid .t-excl .chip, .run-grid .t-nodata .chip, .run-grid .link.t-excl, .run-grid .link.t-nodata, .grid-key.t-excl { fill: var(--grid-excl); stroke: var(--grid-excl); background: var(--grid-excl); } +.grid-key { display: inline-block; width: 18px; height: 4px; border-radius: 2px; } +.grid-legend { margin: 0 0 10px; } +.run-grid a, .run-grid a:hover { text-decoration: none; } + +/* --- run cycles and the weekly digest --------------------------------------- */ + +.verdict { display: inline-flex; align-items: center; gap: 6px; font-weight: 600; white-space: nowrap; } +.verdict .vdot { width: 10px; height: 10px; border-radius: 50%; flex: none; } +.v-regressed .vdot { background: var(--v-regressed); } +.v-mixed .vdot { background: var(--v-mixed); } +.v-improved .vdot { background: var(--v-improved); } +.v-steady .vdot { background: var(--v-steady); } +.v-first .vdot { background: var(--v-first); } +.v-nodata .vdot { background: var(--v-nodata); } + +.spark { display: block; overflow: visible; } +.spark-line { fill: none; stroke: var(--ink-2); stroke-width: 1.6; stroke-linejoin: miter; } +.spark-dot { fill: var(--ink-2); stroke: var(--surface); stroke-width: 1.2; } +.spark-now { fill: var(--ink); stroke: var(--surface); stroke-width: 2; } + +table.board td, table.weekly td { vertical-align: top; } +table.board .big { font-size: 22px; font-weight: 650; font-variant-numeric: tabular-nums; } +table.board .sub, table.weekly .sub { color: var(--ink-muted); font-size: 12px; margin: 2px 0 0; } +table.weekly .env-name { font-size: 15px; font-weight: 650; text-align: left; white-space: nowrap; } +table.weekly .entry + .entry { margin-top: 10px; } +table.weekly .headline { margin-top: 2px; white-space: nowrap; } +table.weekly .value { font-size: 18px; font-weight: 650; color: var(--ink); font-variant-numeric: tabular-nums; } +table.weekly .who { font-weight: 600; margin-right: 4px; } +table.weekly .change { color: var(--ink-2); margin-left: 6px; } +ul.flags { margin: 0; padding-left: 18px; } +ul.flags li { margin: 4px 0; } + +/* --- live updates ------------------------------------------------------------ */ + +.live-note { color: var(--ink-muted); font-size: 12px; margin: -12px 0 12px; } +.badge.live { display: inline-flex; align-items: center; gap: 5px; } +.pulse { width: 7px; height: 7px; border-radius: 50%; background: var(--status-passed); animation: pulse 1.6s ease-in-out infinite; } +@keyframes pulse { 0%, 100% { opacity: 1; } 50% { opacity: 0.3; } } +@media (prefers-reduced-motion: reduce) { .pulse { animation: none; } } +.progress-card { padding-top: 14px; padding-bottom: 12px; } +.progress-head { display: flex; justify-content: space-between; gap: 12px; margin-bottom: 6px; font-variant-numeric: tabular-nums; } +.progress { height: 10px; border-radius: 5px; background: var(--surface-2); overflow: hidden; } +.progress span { display: block; height: 100%; border-radius: 5px; background: var(--accent); transition: width 0.4s ease; } +.progress-card .sub { margin: 8px 0 0; } + +.callout { border: 1px solid var(--v-mixed); border-left-width: 4px; background: var(--surface); border-radius: 8px; padding: 10px 14px; margin: 0 0 16px; color: var(--ink); } diff --git a/radiator/templates/_macros.html b/radiator/templates/_macros.html new file mode 100644 index 0000000..50b81c7 --- /dev/null +++ b/radiator/templates/_macros.html @@ -0,0 +1,42 @@ +{% macro agent_head(agent) -%} +{{ agent }} +{%- endmacro %} + +{% macro cell(asset, result, env, href=None) -%} +{% if result is none -%} +– +{%- else -%} + + + {%- if result.rank is not none %}#{{ result.rank }}{% endif -%} + +{%- endif %} +{%- endmacro %} + +{% macro status_legend() -%} +
+ {% for s in STATUSES %}{{ status_badge(s) }}{% endfor %} +
+{%- endmacro %} + +{# Moving between pages of a list, keeping every other parameter (filters). #} +{% macro pager(request, p, anchor='') -%} +{% if p.pages > 1 %} + +{% endif %} +{%- endmacro %} diff --git a/radiator/templates/asset.html b/radiator/templates/asset.html new file mode 100644 index 0000000..a1ba9ad --- /dev/null +++ b/radiator/templates/asset.html @@ -0,0 +1,97 @@ +{% extends "base.html" %} +{% import "_macros.html" as m %} +{% set a = history.latest %} +{% block title %}{{ a.name or a.asset_id }}{% endblock %} +{% block content %} +
+
Runs / {{ history.test_case_id }} / {{ history.asset_id }}
+

{{ a.name or a.asset_id }}

+
+ {{ a.input_curie or '' }}{% if a.output_curie %} → {{ a.output_curie }}{% endif %} + {% if a.predicate %}{{ a.predicate }}{% endif %} + {% if a.expected_output %}expected: {{ a.expected_output }}{% endif %} + {{ a.kind }} +
+
+ +{% if envs|length > 1 %} +
+ + +
+{% endif %} + +
+

Status history

+

The last {{ history.points|length }} runs {{ 'with no recorded environment' if env == NO_ENV else 'in ' ~ env }}, oldest to newest. Click a cell to open that run.

+ {% set n = history.points|length %} +
+
+ {% for agent in history.agents %} +
{{ m.agent_head(agent) }}
+ {% for run, asset in history.points %}{% set r = history.agent_result(asset, agent) %} + {% if r %} + {{ STATUS_LABELS[r.status][0] }} + {% else %} + · + {% endif %} + {% endfor %} + {% endfor %} +
+ {% for run, asset in history.points %} +
{% if loop.first or loop.last %}{{ run.started_at.strftime('%b %-d') }}{% endif %}
+ {% endfor %} +
+
+ {{ m.status_legend() }} +
+ +
+
+

Rank of the expected answer

+

Lower is better; runs where it wasn't found are gaps

+ {{ rank_chart if rank_chart else '

The expected answer has no recorded rank in this history.

'|safe }} +
+
+

Number of results

+

How many results each agent returned

+ {{ count_chart if count_chart else '

Result counts weren\'t recorded for these runs.

'|safe }} +
+
+ +
+
+

{{ 'Every run' if show_all or history.points|length <= 10 else 'Latest 10 runs' }}

+ {% if history.points|length > 10 %} + {% if show_all %}Show the latest 10 + {% else %}Show all {{ history.points|length }}{% endif %} + {% endif %} +
+
+ + + + {% for run, asset in table_runs %} + {% for r in asset.agents %} + + + + + + + + + + + + {% endfor %} + {% endfor %} + +
RunAgentStatusFoundRankScoreResultsTimeResponse
{% if loop.first %}{{ when(run.started_at) }}{% endif %}{{ m.agent_head(r.agent) }}{{ status_badge(r.status) }}{% if r.message %} {{ r.message }}{% endif %}{{ '–' if r.found is none else ('yes' if r.found else 'no') }}{{ r.rank if r.rank is not none else '–' }}{{ '%.3g'|format(r.score) if r.score is not none else '–' }}{{ '{:,}'.format(r.n_results) if r.n_results is not none else '–' }}{{ '%.1fs'|format(r.response_time_s) if r.response_time_s is not none else '–' }}{% if r.pk %}{{ r.pk[:8] }}…{% endif %}
+
+
+{% endblock %} diff --git a/radiator/templates/base.html b/radiator/templates/base.html new file mode 100644 index 0000000..c8d551c --- /dev/null +++ b/radiator/templates/base.html @@ -0,0 +1,28 @@ + + + + + + {% block title %}Information Radiator{% endblock %} · Translator + + + + +
+ Information Radiator + +
+
+ + {% block content %}{% endblock %} +
+ + + + diff --git a/radiator/templates/cycle.html b/radiator/templates/cycle.html new file mode 100644 index 0000000..bbcfa86 --- /dev/null +++ b/radiator/templates/cycle.html @@ -0,0 +1,65 @@ +{% extends "base.html" %} +{% import "_macros.html" as m %} +{% block title %}Run cycle · {{ cycle.env }}{% endblock %} +{% block content %} +
+
Weekly / run cycle
+

{{ (cycle.env or 'mixed environments')|upper }} run cycle · {{ cycle.started_at.strftime('%a %b %-d') }}

+
+ Started {{ when(cycle.started_at) }}{% if cycle.ended_at %}, finished {{ when(cycle.ended_at) }}{% else %}, still running{% endif %} + Each test against its previous run in {{ cycle.env or 'its environment' }} +
+
+ +
+
+ + + + + + {% for row in cycle.rows %} + + + + + + + + + {% else %} + + {% endfor %} + +
TestResultChangeLast {{ history_runs }} runsVerdictFlags
{{ row.label }}
{{ row.run.suite }} · #{{ row.run.number }}
{{ row.value_text }}
{{ row.unit }}
{{ row.change_text }}
{{ row.detail_text }}{% if row.previous_run and row.perf is none %} · what changed{% endif %}
{{ sparkline(row.history, pct=row.perf is none) }}{{ verdict_chip(row.verdict) }}{% if row.flags %}⚑ {{ row.flags|length }}{% else %}–{% endif %}
No results in this cycle yet.
+
+

Regressions and fixes count agent results that went from passing to not passing, or back.

+
+ +{% if cycle.flags %} +
+

Flags

+

Oddities against the previous run: results gone, new errors, slower responses, the expected answer ranked much lower while still passing, and performance drops.

+ +
+{% endif %} + +{% for row in cycle.rows if row.changes %} +
+

{{ row.label }}: regressions and fixes by agent

+

Everything that changed, including rank moves and skips

+ + + + {% for c in row.changes %} + + + + + + {% endfor %} + +
AgentRegressionsFixed
{{ m.agent_head(c.agent) }}{% for a in c.regressions %}{{ a.label }}{{ ', ' if not loop.last }}{% else %}–{% endfor %}{% for a in c.fixed %}{{ a.label }}{{ ', ' if not loop.last }}{% else %}–{% endfor %}
+
+{% endfor %} +{% endblock %} diff --git a/radiator/templates/diff.html b/radiator/templates/diff.html new file mode 100644 index 0000000..9a1c72c --- /dev/null +++ b/radiator/templates/diff.html @@ -0,0 +1,72 @@ +{% extends "base.html" %} +{% import "_macros.html" as m %} +{% block title %}Changes · {{ run.suite }}{% endblock %} +{% block content %} +
+
Runs / {{ run.suite }} · {{ when(run.started_at) }} / changes
+

What changed

+
+ {% if base %} + {{ when(base.started_at) }} → {{ when(run.started_at) }} + {{ run.suite }} · {{ run.env }} + {% else %} + There's no earlier run of this suite in this environment to compare with. + {% endif %} +
+
+{% if trapi_change %} +
TRAPI {{ trapi_change[0] }} → {{ trapi_change[1] }} since the previous run: some of the changes may come from the version change rather than from the services.
+{% endif %} +{% if base and running %} +

This run hasn't finished: these are the changes among the assets tested so far. Reload to see more.

+{% endif %} +{% if base %} +
+ {% for kind, title, _ in kinds %} + {{ diff.count(kind) }}{{ title }} + {% endfor %} +
+ +{% for kind, title, description in kinds %}{% set changes = diff.changes[kind] %} +
+

{{ title }} ({{ changes|length }})

+

{{ description }}

+ {% if changes %} +
+ + + + {% if kind not in ('added', 'removed') %}{% else %}{% endif %} + + + {% for c in changes[:300] %} + + + + {% if kind not in ('added', 'removed') %} + + + + + + {% else %} + + {% endif %} + + {% endfor %} + +
AssetExpectedAgentBeforeAfterRankStatus
{{ c.asset.name or c.asset.asset_id }}{{ c.asset.expected_output or '–' }}{{ m.agent_head(c.agent) }}{{ status_badge(c.before.status) }}→{{ status_badge(c.after.status) }} + {% if c.before.rank is not none or c.after.rank is not none %} + {{ c.before.rank if c.before.rank is not none else '–' }} → {{ c.after.rank if c.after.rank is not none else '–' }} + {% if c.rank_delta %}({{ '%+d'|format(c.rank_delta) }}){% endif %} + {% endif %} + {{ status_badge(c.asset.status) }}
+
+ {% if changes|length > 300 %}

Showing the first 300 of {{ changes|length }}.

{% endif %} + {% else %} +

None.

+ {% endif %} +
+{% endfor %} +{% endif %} +{% endblock %} diff --git a/radiator/templates/grid.html b/radiator/templates/grid.html new file mode 100644 index 0000000..980b085 --- /dev/null +++ b/radiator/templates/grid.html @@ -0,0 +1,36 @@ +{% extends "base.html" %} +{% block title %}Run grid{% endblock %} +{% block content %} +
+

Acceptance run grid

+
Every full run of a suite, by environment over time. A card's trend compares its pass rate (skips left out) with the previous run in the same environment; FLAT is within 0.1 pts.{% if excluded %} Greyed out, and without a trend: {{ excluded|join(', ') }}.{% endif %}
+
+ +
+ + + +
+ +
+ {% if grid_svg %} +
+ {% for t in ['up', 'down', 'flat', 'base', 'excl'] %} + {{ trend_labels[t] }} + {% endfor %} + · click a card to open the run +
+
{{ grid_svg }}
+ {% else %} +

No finished full runs of {{ suite or 'this suite' }} in the last {{ days }} days.

+ {% endif %} +
+{% endblock %} diff --git a/radiator/templates/login.html b/radiator/templates/login.html new file mode 100644 index 0000000..71d4774 --- /dev/null +++ b/radiator/templates/login.html @@ -0,0 +1,23 @@ + + + + + + Log in · Information Radiator + + + + + + + diff --git a/radiator/templates/performance.html b/radiator/templates/performance.html new file mode 100644 index 0000000..149374c --- /dev/null +++ b/radiator/templates/performance.html @@ -0,0 +1,48 @@ +{% extends "base.html" %} +{% block title %}Performance{% endblock %} +{% block content %} +
+

Performance

+
HelmsDeep load tests: the highest concurrency each service sustained within its SLO, and checkpoint verdicts.
+
+ +
+ + +
+ +{% if not panels %} +

No performance runs in the last {{ days }} days.

+{% endif %} +
+{% for key, rows, chart in panels %} +
+

{{ key }}

+ {% set last = rows[-1][1] %} +

Latest: {{ '%.1f'|format(last.max_sustainable_concurrency) if last.max_sustainable_concurrency is not none else 'no stage met the SLO' }} + {% if last.checkpoints_passed is not none %} · checkpoints {{ status_badge('PASSED' if last.checkpoints_passed else 'FAILED') }}{% endif %}

+ {{ chart }} +
+ Table view +
+ + + {% for run, perf in rows|reverse %} + + + + + + + {% endfor %} + +
RunProfileMax concurrencyCheckpoints
{{ when(run.started_at) }}{{ perf.profile or 'default' }}{{ '%.1f'|format(perf.max_sustainable_concurrency) if perf.max_sustainable_concurrency is not none else '–' }}{% if perf.error %}{{ status_badge('ERROR') }}{% elif perf.checkpoints_passed is none %}none{% else %}{{ status_badge('PASSED' if perf.checkpoints_passed else 'FAILED') }}{% endif %}
+
+
+{% endfor %} +
+{% endblock %} diff --git a/radiator/templates/run.html b/radiator/templates/run.html new file mode 100644 index 0000000..016bdcb --- /dev/null +++ b/radiator/templates/run.html @@ -0,0 +1,229 @@ +{% extends "base.html" %} +{% import "_macros.html" as m %} +{% block title %}{{ run.suite }} · {{ when(run.started_at) }}{% endblock %} +{% block live %}{% if progress.running %} data-live="/api/runs/{{ run.id }}/progress" data-live-interval="10" data-live-version="{{ progress.version }}" data-live-until-done{% endif %}{% endblock %} +{% block content %} +
+
Runs / {{ run.suite }}
+

#{{ run.number }} {{ run.suite }} · {{ run.env or 'no env' }}

+
+ {{ when(run.started_at) }} + {% if run.ended_at %}Duration {{ duration(run) }}{% else %}{{ run_badge(run) }} for {{ elapsed(run) }}{% endif %} + {% if run.target %}target: {{ run.target }}{% endif %} + {% if run.query_type %}{{ run.query_type }} only{% endif %} + {% if run.trapi_version %}TRAPI {{ trapi_minor(run.trapi_version) }}{% endif %} + {% if run.query_parameters %}query parameters: {{ params_text(run.query_parameters) }}{% endif %} + {% if run.harness_version %}harness {{ run.harness_version }}{% endif %} + {% if run.origin == 'zebrunner_import' %}imported from Zebrunner{% endif %} +
+
+ + +{% if trapi_change %} +
TRAPI {{ trapi_change[0] }} → {{ trapi_change[1] }} since the previous run: some of the changes may come from the version change rather than from the services.
+{% endif %} + +
+{% if progress.running %} + {% set done, total = progress.done_assets, progress.expected_assets %} +
+ {% if total %} +
{{ done }} of {{ total }} assets{{ (100 * done / total)|round|int if done <= total else 100 }}%
+
+ {% else %} +
{{ done }} assets so far
+ {% endif %} +

Results arrive as each test case finishes; this page updates by itself.

+
+{% endif %} +
+ +
+
+
Pass rate{% if progress.running %} so far{% endif %}
+
{{ pct(rate) }}
+ {% if previous_rate is not none and rate is not none %} + {% set d = (rate - previous_rate) * 100 %} +
+ {{ '%+.1f'|format(d) }} pts vs previous run +
+ {% endif %} +
+
+
Assets tested
+
{{ overall.values()|sum }}
+
{{ overall.get('PASSED', 0) }} passed · {{ overall.get('SKIPPED', 0) }} skipped
+
+
+
Regressions
+
{% if previous %}{{ diff.count('regression') }}{% else %}–{% endif %}
+
agent results that stopped passing{% if progress.running %}, so far{% endif %}
+
+
+
Fixed
+
{% if previous %}{{ diff.count('fixed') }}{% else %}–{% endif %}
+
{% if previous %}vs {{ when(previous.started_at) }}{% else %}no earlier run to compare{% endif %}
+
+
+ +
+
+
+

By agent

Each agent's results across every asset in this run

+ {% if previous %}What changed since the last run →{% endif %} +
+
+ + + {% for s in STATUSES %}{% endfor %} + + {% for agent in agents %}{% set c = agent_counts[agent] %} + + + + + {% for s in STATUSES %}{% endfor %} + + {% endfor %} + +
AgentResultsPass rate{{ STATUS_LABELS[s][1] }}
{{ m.agent_head(agent) }}{{ status_bar(c) }}{{ pct(pass_rate(c)) }}{% if c.get(s) %}{{ c.get(s) }}{% else %}0{% endif %}
+
+ {{ m.status_legend() }} +
+
+ +
+{% if agents %} +
+

By agent and expected output

+

Each agent's results split by what the test expects of the answer. For NeverShow, passing means the answer stayed out of the results.

+
+ + + {% for s in STATUSES %}{% endfor %} + {% for agent in agents %}{% set rows = agent_expected_counts.get(agent, []) %} + + {% for expected, c in rows %} + + {% if loop.first %}{% endif %} + + + + {% for s in STATUSES %}{% endfor %} + + {% endfor %} + + {% endfor %} +
AgentExpected outputResultsPass rate{{ STATUS_LABELS[s][1] }}
{{ m.agent_head(agent) }}{{ expected or 'Not specified' }}{{ status_bar(c) }}{{ pct(pass_rate(c)) }} + {%- if c.get(s) and expected -%} + {{ c.get(s) }} + {%- elif c.get(s) -%}{{ c.get(s) }} + {%- else -%}0{%- endif -%} +
+
+
+{% endif %} +
+ +
+{% if run.performance %} +
+

Performance

+

HelmsDeep runs in this test run

+
+ + + + {% for perf in run.performance %} + + + + + + + {% endfor %} + +
HostRun typeProfileResultMax sustainable concurrencyCheckpoints
{{ perf.host }}{{ perf.helmsdeep_target or '–' }}{{ perf.profile or 'default' }}{{ status_badge(perf.status) }}{% if perf.error %} {{ perf.error }}{% endif %}{{ '%.1f'|format(perf.max_sustainable_concurrency) if perf.max_sustainable_concurrency is not none else '–' }}{% if perf.knee_unsupported %} unsupported{% endif %}{% if perf.checkpoints_passed is none %}none configured{% else %}{{ status_badge('PASSED' if perf.checkpoints_passed else 'FAILED') }}{% endif %}
+
+
+{% endif %} +
+ +
+
+

Results

Every asset against every agent. The number is the rank the expected answer came back at.

+
+
+ + + + + {# a new filter starts again at page 1, at the same page size #} + {% if request.query_params.get('per_page') %}{% endif %} + + {% if filtered %}Clear{% endif %} +
+
+ {% if assets %} + {% set p = results_page %} +
+ Showing {{ p.start }}–{{ p.end }} of {{ p.total }} asset{{ '' if p.total == 1 else 's' }}{% if filtered %} matching{% endif %} +
+ {% for key in ['status', 'agent', 'expected', 'q'] %}{% if filters[key] %}{% endif %}{% endfor %} + + +
+ {{ m.pager(request, p, '#matrix') }} +
+
+ + + + {% for a in agents %}{% endfor %} + + + {% for asset in assets %} + {% set by_agent = {} %}{% for r in asset.agents %}{% set _ = by_agent.update({r.agent: r}) %}{% endfor %} + + + + + {% for a in agents %}{% set r = by_agent.get(a) %} + + {% endfor %} + + {% endfor %} + +
AssetExpectedOverall{{ m.agent_head(a) }}
{{ asset.name or asset.asset_id }} +
{{ asset.input_curie or '' }}{% if asset.output_curie %} → {{ asset.output_curie }}{% endif %}
{{ asset.expected_output or '–' }}{{ status_badge(asset.status) }}{{ m.cell(asset, r, run.env, pk_url(r.pk) if r else None) }}
+
+
+

{% if filtered %}{{ p.total }} of {{ run.assets|length }} assets match. {% endif %}Hover a cell for details; click it to open the response.

+ {{ m.pager(request, p, '#matrix') }} +
+ {% else %} +

{% if run.assets %}No assets match these filters.{% elif progress.running %}No results yet: they arrive as each test case finishes.{% else %}No asset results in this run.{% endif %}

+ {% endif %} +
+
+{% endblock %} diff --git a/radiator/templates/runs.html b/radiator/templates/runs.html new file mode 100644 index 0000000..e9af816 --- /dev/null +++ b/radiator/templates/runs.html @@ -0,0 +1,93 @@ +{% extends "base.html" %} +{% import "_macros.html" as m %} +{% block title %}Runs{% endblock %} +{% block live %} data-live="/api/live" data-live-interval="20" data-live-version="{{ live_version }}"{% endblock %} +{% block content %} +
+

Test runs

+
{{ total }} run{{ '' if total == 1 else 's' }}{% if suite or env %} matching{% endif %}
+
+ +
+ + + {# a new filter starts again at page 1, at the same page size #} + {% if request.query_params.get('per_page') %}{% endif %} + + {% if suite or env %}Clear{% endif %} +
+ + +
+ {% if runs %} + {% set p = runs_page %} +
+ Showing {{ p.start }}–{{ p.end }} of {{ p.total }} run{{ '' if p.total == 1 else 's' }}, newest first +
+ {% if suite %}{% endif %} + {% if env %}{% endif %} + + +
+ {{ m.pager(request, p) }} +
+
+ + + + + + + {% for run in runs %} + {% set c = counts[run.id] %}{% set pc = perf_counts.get(run.id) %} + + + + + + {% if pc and not c %} + + + + {% else %} + + + + {% endif %} + + + + {% endfor %} + +
RunStartedSuiteEnvResultsPass rateAssetsDuration
#{{ run.number }}{{ when(run.started_at) }}
{{ ago(run.started_at) }}		{{ run.suite }} + {% if run.target %}target: {{ run.target }}{% endif %} + {% if run.query_type %}{{ run.query_type }} only{% endif %} + {% if run.trapi_version %}TRAPI {{ trapi_minor(run.trapi_version) }}{% endif %} + {% if run.query_parameters %}parameters{% endif %} + {{ run.env or '–' }}{{ status_bar(pc) }}performance · {{ pc.values()|sum }} host{{ '' if pc.values()|sum == 1 else 's' }}{{ pct(pass_rate(pc)) }}–{{ status_bar(c) }}{{ pct(pass_rate(c)) }}{{ c.values()|sum }}{% if run.ended_at %}{{ duration(run) }}{% else %}{{ run_badge(run) }}{% if run_state(run) == 'running' %}
{{ elapsed(run) }}{% endif %}{% endif %}		{% if run.origin == 'zebrunner_import' %}imported{% endif %}
+
+ {% if p.pages > 1 %} +
+ + {{ m.pager(request, p) }} +
+ {% endif %} + {% else %} +

No runs yet. They appear here as soon as the harness starts one.

+ {% endif %} +
+{% endblock %} diff --git a/radiator/templates/trends.html b/radiator/templates/trends.html new file mode 100644 index 0000000..ca4bee3 --- /dev/null +++ b/radiator/templates/trends.html @@ -0,0 +1,64 @@ +{% extends "base.html" %} +{% import "_macros.html" as m %} +{% block title %}Trends{% endblock %} +{% block content %} +
+

Trends

+
Pass rate per agent over full runs of a suite: skips are left out, and runs with a target override or a single query type aren't included.
+
+ +
+ + + + +
+ +{% if not points %} +

No full runs of {{ suite or 'this suite' }} in the last {{ days }} days.

+{% else %} +
+

Pass rate by agent

+

{{ points|length }} runs · all expected outputs

+
{{ overall_chart }}
+
{{ overall_chart_narrow }}
+
+ Table view +
+ + {% for a in agents %}{% endfor %} + + {% for p in points|reverse %} + + {% for a in agents %}{% endfor %} + {% endfor %} + +
Run{{ m.agent_head(a) }}
{{ when(p.run.started_at) }}{{ pct(rate(p, a), 1) }}
+
+
+
+ +
+

By expected output

+

The same pass rate, split by what the test expects of the answer

+
+ {% for expected, chart in multiples %} +

{{ expected }}

{{ chart }}
+ {% endfor %} +
+
+{% endif %} +{% endblock %} diff --git a/radiator/templates/weekly.html b/radiator/templates/weekly.html new file mode 100644 index 0000000..b1a2dc4 --- /dev/null +++ b/radiator/templates/weekly.html @@ -0,0 +1,85 @@ +{% extends "base.html" %} +{% block title %}Weekly{% endblock %} +{% block live %}{% if not next_week %} data-live="/api/live" data-live-interval="60" data-live-version="{{ live_version }}"{% endif %}{% endblock %} +{% block content %} +
+

Weekly test status

+
+ {{ digest.since.strftime('%b %-d') }} – {{ digest.until.strftime('%b %-d') }} + Each environment's latest full runs, against the run before + ← previous week{% if next_week %} · next week →{% endif %} +
+
+ + +
+
+ {% if digest.envs %} +
+ + + + {% for kind in kinds %}{% endfor %} + + + + {% for env in digest.envs %} + + + {% for kind in kinds %}{% set rows = env.cell(kind) %} + + {% endfor %} + + + {% endfor %} + +
Environment{{ kind_labels[kind] }}Flags
+ {{ (env.env or 'no environment')|upper }} + {% for cid in env.cycle_ids %}
run cycle
{% endfor %} + 		+ {% for row in rows %} +
+
{{ verdict_chip(row.verdict) }}
+
+ {% if row.perf %}{{ row.perf.helmsdeep_target or row.perf.host }}{% endif %} + {{ row.value_text }} + {{ row.change_text }} +
+
{{ row.detail_text }} · #{{ row.run.number }}, {{ row.run.started_at.strftime('%a %b %-d') }}{% if row.previous_run and row.perf is none %} · what changed{% endif %}
+
+ {% else %}not run this week{% endfor %} + 		{% if env.flags %}⚑ {{ env.flags|length }}{% else %}–{% endif %}
+
+ {% else %} +

No finished runs in this week.

+ {% endif %} +
+ +{% for env in digest.envs if env.flags %} +
+

Flags · {{ (env.env or 'no environment')|upper }}

+

Oddities against the previous run: results gone, new errors, slower responses, the expected answer ranked much lower while still passing, and performance drops.

+ +
+{% endfor %} +
+ +
+

Recent run cycles

+

Acceptance, pathfinder and performance, back to back in one environment.

+ {% if recent %} + + + + {% for cid, env, started in recent %} + + {% endfor %} + +
StartedEnvironment
{{ when(started) }}{{ env or '–' }}
+ {% else %} +

No run cycles yet. See test-harness-cycle in the README.

+ {% endif %} +
+{% endblock %} diff --git a/radiator/tests/__init__.py b/radiator/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/radiator/tests/conftest.py b/radiator/tests/conftest.py new file mode 100644 index 0000000..1e4a536 --- /dev/null +++ b/radiator/tests/conftest.py @@ -0,0 +1,84 @@ +"""Fixtures: a migrated Postgres database and an app pointed at it. + +Needs a Postgres the tests may wipe, given as RADIATOR_TEST_DATABASE_URL, eg +postgresql+psycopg://postgres@localhost:5432/radiator_test +""" + +import os + +import pytest +from alembic import command +from alembic.config import Config +from fastapi.testclient import TestClient +from sqlalchemy import create_engine, text + +from radiator.app import create_app +from radiator.config import Settings + +HERE = os.path.dirname(__file__) +TOKEN = "test-token" +USERNAME = "translator" +PASSWORD = "correct horse battery staple" + + +@pytest.fixture(scope="session") +def database_url(): + url = os.getenv("RADIATOR_TEST_DATABASE_URL") + if not url: + pytest.skip("RADIATOR_TEST_DATABASE_URL is not set") + engine = create_engine(url) + with engine.begin() as conn: + conn.execute(text("DROP SCHEMA public CASCADE; CREATE SCHEMA public")) + engine.dispose() + config = Config(os.path.join(HERE, "..", "alembic.ini")) + config.attributes["database_url"] = url + command.upgrade(config, "head") + return url + + +@pytest.fixture +def settings(database_url): + return Settings( + database_url=database_url, + api_token=TOKEN, + username=USERNAME, + password=PASSWORD, + session_secret="test-secret", + secure_cookies=False, + ) + + +@pytest.fixture +def app(settings): + app = create_app(settings) + yield app + with app.state.sessionmaker() as session: + session.execute( + text("TRUNCATE runs, asset_results, agent_results, performance_results") + ) + session.commit() + + +@pytest.fixture +def client(app): + """Unauthenticated.""" + return TestClient(app) + + +@pytest.fixture +def api(app): + """Authenticated with the API token.""" + return TestClient(app, headers={"Authorization": f"Bearer {TOKEN}"}) + + +@pytest.fixture +def browser(app): + """Logged in through the login form, like a person.""" + client = TestClient(app) + res = client.post( + "/login", + data={"username": USERNAME, "password": PASSWORD}, + follow_redirects=False, + ) + assert res.status_code == 303 + return client diff --git a/radiator/tests/factories.py b/radiator/tests/factories.py new file mode 100644 index 0000000..24fdf7e --- /dev/null +++ b/radiator/tests/factories.py @@ -0,0 +1,53 @@ +"""Builders for schema payloads in tests and the demo seed.""" + +import uuid +from datetime import datetime, timedelta, timezone + +import radiator_schema as schema + + +def run_create(started_at=None, **kwargs): + defaults = dict( + run_id=uuid.uuid4(), + suite="sprint_4_tests", + env="ci", + started_at=started_at or datetime.now(timezone.utc), + ) + defaults.update(kwargs) + return schema.RunCreate(**defaults) + + +def agent(name, status="PASSED", **kwargs): + return schema.AgentResult(agent=name, status=status, **kwargs) + + +def asset( + asset_id="Asset_1", + test_case_id="TestCase_1", + status="PASSED", + agents=None, + **kwargs, +): + defaults = dict( + kind="acceptance", + name=f"{asset_id} name", + expected_output="TopAnswer", + predicate="biolink:treats", + input_curie="MONDO:0004979", + output_curie="CHEBI:31690", + ) + defaults.update(kwargs) + return schema.AssetResult( + test_case_id=test_case_id, + asset_id=asset_id, + status=status, + agents=agents if agents is not None else [agent("ars", status)], + **defaults, + ) + + +def finish(started_at=None, minutes=42, counts=None): + started_at = started_at or datetime.now(timezone.utc) + return schema.RunFinish( + ended_at=started_at + timedelta(minutes=minutes), counts=counts or {} + ) diff --git a/radiator/tests/test_auth.py b/radiator/tests/test_auth.py new file mode 100644 index 0000000..3205095 --- /dev/null +++ b/radiator/tests/test_auth.py @@ -0,0 +1,125 @@ +"""Nothing is reachable without logging in or a token.""" + +import pytest + +from radiator.app import _safe_next + +from .conftest import PASSWORD, USERNAME + + +def test_health_check_needs_no_auth(client): + assert client.get("/healthz").json() == {"ok": True} + + +@pytest.mark.parametrize("path", ["/", "/trends", "/performance"]) +def test_pages_redirect_to_login(client, path): + res = client.get(path, follow_redirects=False) + assert res.status_code == 303 + assert res.headers["location"] == f"/login?next={path}" + + +def test_read_api_needs_token_or_login(client, api, browser): + assert client.get("/api/runs").status_code == 401 + assert api.get("/api/runs").status_code == 200 + assert browser.get("/api/runs").status_code == 200 + + +def test_ingest_needs_the_token(client, browser): + body = { + "run_id": "6e3c1d8e-5f0e-4bb8-9d7e-1f1e0b6f0a11", + "suite": "s", + "started_at": "2026-10-06T00:00:00Z", + } + assert client.post("/api/ingest/runs", json=body).status_code == 401 + bad = client.post( + "/api/ingest/runs", json=body, headers={"Authorization": "Bearer nope"} + ) + assert bad.status_code == 401 + # a logged-in browser still can't write: ingest is token-only + assert browser.post("/api/ingest/runs", json=body).status_code == 401 + + +def test_login_and_logout(client): + res = client.post( + "/login", + data={"username": USERNAME, "password": PASSWORD, "next": "/trends"}, + follow_redirects=False, + ) + assert res.status_code == 303 + assert res.headers["location"] == "/trends" + assert client.get("/", follow_redirects=False).status_code == 200 + + client.post("/logout", follow_redirects=False) + assert client.get("/", follow_redirects=False).status_code == 303 + + +def test_wrong_password(client): + res = client.post("/login", data={"username": USERNAME, "password": "nope"}) + assert res.status_code == 401 + assert "didn" in res.text + assert client.get("/", follow_redirects=False).status_code == 303 + + +def test_login_is_throttled(client): + for _ in range(10): + client.post("/login", data={"username": USERNAME, "password": "nope"}) + # even the right password is refused once the address is throttled + res = client.post("/login", data={"username": USERNAME, "password": PASSWORD}) + assert res.status_code == 401 + assert "Too many" in res.text + + +@pytest.mark.parametrize( + "next_url, expected", + [ + (None, "/"), + ("/runs/abc?x=1", "/runs/abc?x=1"), + ("https://evil.example", "/"), + ("//evil.example", "/"), + ("/\\evil.example", "/"), + ("javascript:alert(1)", "/"), + ], +) +def test_login_only_redirects_on_site(next_url, expected): + assert _safe_next(next_url) == expected + + +def test_throttle_trusts_only_the_ingress_entry(client): + """Forging X-Forwarded-For can't dodge the throttle: only the last entry, + the one the ingress adds, counts.""" + for i in range(10): + client.post( + "/login", + data={"username": USERNAME, "password": "nope"}, + headers={"X-Forwarded-For": f"10.0.0.{i}, 203.0.113.7"}, + ) + res = client.post( + "/login", + data={"username": USERNAME, "password": PASSWORD}, + headers={"X-Forwarded-For": "198.51.100.1, 203.0.113.7"}, + ) + assert res.status_code == 401 and "Too many" in res.text + # someone else, behind the same ingress, is unaffected + other = client.post( + "/login", + data={"username": USERNAME, "password": PASSWORD}, + headers={"X-Forwarded-For": "203.0.113.8"}, + follow_redirects=False, + ) + assert other.status_code == 303 + + +def test_throttle_memory_is_bounded(): + from radiator.auth import LoginThrottle + + throttle = LoginThrottle(max_clients=100, window_s=0) + for i in range(1000): + throttle.failed(f"client-{i}") + assert len(throttle._failures) <= 101 + + +def test_login_returns_to_the_full_url(client): + res = client.get("/runs/abc?status=FAILED&agent=arax", follow_redirects=False) + target = res.headers["location"] + page = client.get(target) + assert 'value="/runs/abc?status=FAILED&agent=arax"' in page.text diff --git a/radiator/tests/test_cycles.py b/radiator/tests/test_cycles.py new file mode 100644 index 0000000..6582802 --- /dev/null +++ b/radiator/tests/test_cycles.py @@ -0,0 +1,344 @@ +"""Run cycles: which test a run is, flags, verdicts, the board and the digest.""" + +import uuid +from datetime import datetime, timedelta, timezone + +import radiator_schema as schema +from radiator import cycles +from radiator.ingest import ingest_payload + +from . import factories as f + +T0 = datetime(2026, 9, 7, 6, tzinfo=timezone.utc) # a Monday + + +def _assets(statuses, kind="acceptance", **agent_kw): + """One asset per status, each answered by the ARS and aragorn alike.""" + out = [] + for i, status in enumerate(statuses): + kw = {k: v[i] if isinstance(v, list) else v for k, v in agent_kw.items()} + out.append( + f.asset( + f"A{i}", + status=status, + kind=kind, + agents=[f.agent("ars", status), f.agent("aragorn", status, **kw)], + ) + ) + return out + + +def _results_run(started, statuses, env="ci", cycle_id=None, kind="acceptance", **kw): + suite = "pathfinder_tests" if kind == "pathfinder" else "sprint_4_tests" + return schema.RunPayload( + run=f.run_create(started_at=started, env=env, suite=suite, cycle_id=cycle_id), + results=_assets(statuses, kind=kind, **kw), + finish=f.finish(started), + ) + + +def _perf_run( + started, msc, env="ci", cycle_id=None, target="ars", passed=True, error=None +): + return schema.RunPayload( + run=f.run_create( + started_at=started, + env=env, + suite="performance_tests", + target=target, + target_url=f"https://{target}.{env}.transltr.io", + cycle_id=cycle_id, + ), + performance=[ + schema.PerformanceResult( + test_case_id="Perf_1", + asset_id="Perf_mixed", + host=f"https://{target}.{env}.transltr.io", + helmsdeep_target=target, + profile="mixed", + status="PASSED" if passed else "FAILED", + error=error, + max_sustainable_concurrency=msc, + checkpoints_passed=passed, + ) + ], + finish=f.finish(started, minutes=68), + ) + + +def _load(app, *payloads): + with app.state.sessionmaker() as session: + for payload in payloads: + ingest_payload(session, payload) + session.commit() + + +def _cycle(app, start, env="ci", acceptance=None, pathfinder=None, msc=None, **kw): + cycle_id = uuid.uuid4() + payloads = [] + if acceptance is not None: + payloads.append(_results_run(start, acceptance, env, cycle_id, **kw)) + if pathfinder is not None: + payloads.append( + _results_run( + start + timedelta(hours=1), pathfinder, env, cycle_id, "pathfinder" + ) + ) + if msc is not None: + payloads.append( + _perf_run(start + timedelta(hours=1, minutes=20), msc, env, cycle_id) + ) + _load(app, *payloads) + return cycle_id + + +def _get_cycle(app, cycle_id): + with app.state.sessionmaker() as session: + return cycles.get_cycle(session, cycle_id) + + +P, F = "PASSED", "FAILED" + + +def test_a_cycle_board_compares_each_test_with_its_previous_run(app): + _cycle(app, T0, acceptance=[P, P, F, F], pathfinder=[P, F], msc=10.0) + later = _cycle( + app, + T0 + timedelta(days=7), + acceptance=[P, F, P, P], # A1 regressed; A2, A3 fixed + pathfinder=[P, F], + msc=7.0, + ) + cycle = _get_cycle(app, later) + assert cycle.env == "ci" + assert [row.kind for row in cycle.rows] == [ + "acceptance", + "pathfinder", + "performance", + ] + acceptance, pathfinder, performance = cycle.rows + + assert acceptance.value == 0.75 and acceptance.previous == 0.5 + assert acceptance.change_text == "▲25.0 pts" + # counted per agent result: the ARS and aragorn both flipped + assert (acceptance.regressions, acceptance.fixed) == (2, 4) + assert acceptance.verdict == cycles.IMPROVED + assert [c.agent for c in acceptance.changes] == ["ars", "aragorn"] + assert [a.asset_id for a in acceptance.changes[0].regressions] == ["A1"] + assert acceptance.history == [0.5, 0.75] + + assert pathfinder.verdict == cycles.STEADY + assert pathfinder.change_text == "no change" + + assert performance.label == "Performance · ars" + assert performance.value_text == "7.0" + assert performance.change_text == "▼3.0 (-30%)" + assert performance.verdict == cycles.REGRESSED + assert [flag.kind for flag in performance.flags] == ["concurrency_drop"] + + +def test_verdicts(app): + first = _cycle(app, T0, acceptance=[P, F]) + assert _get_cycle(app, first).rows[0].verdict == cycles.FIRST + for week, (statuses, verdict) in enumerate( + [ + ([F, P], cycles.MIXED), # one regression and one fix + ([F, F], cycles.REGRESSED), + ([P, P], cycles.IMPROVED), + ([P, P], cycles.STEADY), + ], + start=1, + ): + cycle_id = _cycle(app, T0 + timedelta(days=7 * week), acceptance=statuses) + assert _get_cycle(app, cycle_id).rows[0].verdict == verdict, verdict + + +def test_a_cycle_where_everything_was_skipped_has_no_result(app): + _cycle(app, T0, acceptance=[P, P]) + cycle_id = _cycle(app, T0 + timedelta(days=7), acceptance=["SKIPPED", "SKIPPED"]) + row = _get_cycle(app, cycle_id).rows[0] + assert row.verdict == cycles.NO_DATA + assert row.value_text == "all skipped" + + +def test_performance_verdicts_and_flags(app): + _cycle(app, T0, msc=10.0) + for days, msc, verdict, flags in [ + (7, 10.5, cycles.STEADY, []), + (14, 12.0, cycles.IMPROVED, []), + (21, 8.0, cycles.REGRESSED, ["concurrency_drop"]), # -33% + ]: + cycle_id = _cycle(app, T0 + timedelta(days=days), msc=msc) + row = _get_cycle(app, cycle_id).rows[0] + assert row.verdict == verdict, days + assert [f.kind for f in row.flags] == flags, days + + cycle_id = uuid.uuid4() + _load( + app, + _perf_run( + T0 + timedelta(days=28), None, cycle_id=cycle_id, passed=False, error="boom" + ), + ) + row = _get_cycle(app, cycle_id).rows[0] + assert row.verdict == cycles.NO_DATA + assert row.value_text == "no result" + assert [f.kind for f in row.flags] == ["perf_failed", "checkpoints"] + assert "run failed: boom (was 8.0)" in row.flags[0].text + + +def _flags(app, before, after, at=T0): + """The flags between two runs. ``before`` and ``after`` give aragorn's + status and fields per asset; the ARS passes throughout.""" + + def payload(started, spec): + fields = {k: v for k, v in spec.items() if k != "status"} + run = _results_run(started, [P] * len(spec["status"]), **fields) + for asset, status in zip(run.results, spec["status"]): + asset.agents[1] = asset.agents[1].model_copy( + update={"status": schema.Status(status)} + ) + return run + + a, b = payload(at, before), payload(at + timedelta(days=7), after) + _load(app, a, b) + with app.state.sessionmaker() as session: + row = cycles.rows_for(session, [session.get(cycles.Run, b.run.run_id)])[0] + return {flag.kind: flag for flag in row.flags} + + +def test_flags_no_results_on_assets_that_had_them(app): + flags = _flags( + app, + {"status": [P] * 4, "n_results": [100] * 4}, + {"status": ["NO_RESULTS"] * 3 + [P], "n_results": [0, 0, 0, 100]}, + ) + assert set(flags) == {"no_results"} + assert flags["no_results"].who == "aragorn" + assert "no results on 3 assets" in flags["no_results"].text + + +def test_flags_need_enough_assets(app): + flags = _flags( + app, + {"status": [P] * 4, "n_results": [100] * 4}, + {"status": ["NO_RESULTS"] * 2 + [P, P], "n_results": [0, 0, 100, 100]}, + ) + assert flags == {} + + +def test_flags_new_errors(app): + flags = _flags( + app, + {"status": [P] * 3, "http_status": [200] * 3}, + {"status": ["ERROR"] * 3, "http_status": [502] * 3}, + ) + assert set(flags) == {"errors"} + + +def test_flags_slower_responses(app): + flags = _flags( + app, + {"status": [P] * 3, "response_time_s": [4.0, 5.0, 6.0]}, + {"status": [P] * 3, "response_time_s": [14.0, 15.0, 16.0]}, + ) + assert flags["slower"].text == "aragorn: median response time 5.0s → 15.0s" + # doubling a fast response isn't news + assert ( + _flags( + app, + {"status": [P] * 3, "response_time_s": [1.0] * 3}, + {"status": [P] * 3, "response_time_s": [3.0] * 3}, + at=T0 + timedelta(days=14), + ) + == {} + ) + + +def test_flags_rank_drops_while_passing(app): + flags = _flags( + app, + {"status": [P, P], "rank": [2, 2]}, + {"status": [P, P], "rank": [40, 20]}, + ) + assert set(flags) == {"rank_drop"} + assert "on 1 asset, still passing: A0 name (2 → 40)" in flags["rank_drop"].text + + +def test_digest_takes_each_environments_latest_full_runs(app): + week_start = T0 + timedelta(days=7) + _cycle(app, T0, "ci", acceptance=[P, F], msc=10.0) + _cycle(app, week_start, "dev", acceptance=[F, F], pathfinder=[P]) + ci = _cycle(app, week_start + timedelta(days=1), "ci", acceptance=[P, P], msc=11.0) + _load( + app, + # a slice and an override run aren't the environment's status + _results_run(week_start + timedelta(days=2), [F, F], "ci"), + _perf_run(week_start + timedelta(days=2), 6.0, "ci", target="arax"), + ) + with app.state.sessionmaker() as session: + # mark the later ci acceptance run as a query-type slice + run = session.scalars( + cycles.select(cycles.Run).where( + cycles.Run.started_at == week_start + timedelta(days=2), + cycles.Run.suite == "sprint_4_tests", + ) + ).one() + run.query_type = "MVP1" + session.commit() + d = cycles.digest(session, until=week_start + timedelta(days=6), days=7) + assert [e.env for e in d.envs] == ["dev", "ci"] + dev, ci_env = d.envs + assert [r.kind for r in dev.rows] == ["acceptance", "pathfinder"] + assert ci_env.cycle_ids == [ci] + assert [r.label for r in ci_env.rows] == [ + "Acceptance", + "Performance · ars", + "Performance · arax", + ] + assert ci_env.cell("acceptance")[0].value == 1.0 + assert ci_env.cell("acceptance")[0].verdict == cycles.IMPROVED + + +def test_cycle_api_and_pngs(app, api): + cycle_id = _cycle(app, T0, acceptance=[P, F], pathfinder=[P], msc=10.0) + res = api.get(f"/api/cycles/{cycle_id}") + assert res.status_code == 200 + body = res.json() + assert body["env"] == "ci" + assert [r["label"] for r in body["rows"]] == [ + "Acceptance", + "Pathfinder", + "Performance · ars", + ] + assert body["rows"][0]["verdict_label"] == "First run" + assert api.get("/api/cycles").json()[0]["cycle_id"] == str(cycle_id) + png = api.get(f"/api/cycles/{cycle_id}/board.png") + assert png.headers["content-type"] == "image/png" and png.content[:4] == b"\x89PNG" + assert api.get(f"/api/cycles/{uuid.uuid4()}").status_code == 404 + until = (T0 + timedelta(days=1)).isoformat() + digest = api.get("/api/digest", params={"until": until}).json() + assert [e["env"] for e in digest["envs"]] == ["ci"] + png = api.get("/api/digest.png", params={"until": until}) + assert png.content[:4] == b"\x89PNG" + + +def test_cycle_and_weekly_pages(app, browser, client): + cycle_id = _cycle(app, T0, acceptance=[P, F], pathfinder=[P], msc=10.0) + page = browser.get(f"/cycles/{cycle_id}") + assert page.status_code == 200 + assert "CI run cycle" in page.text and "First run" in page.text + weekly = browser.get( + "/weekly", params={"week_of": (T0 + timedelta(days=2)).date().isoformat()} + ) + assert weekly.status_code == 200 + assert f"/cycles/{cycle_id}" in weekly.text + assert browser.get(f"/cycles/{uuid.uuid4()}").status_code == 404 + assert client.get("/weekly", follow_redirects=False).status_code in (302, 303, 401) + + +def test_runs_keep_their_cycle_id(app, api): + cycle_id = _cycle(app, T0, acceptance=[P]) + with app.state.sessionmaker() as session: + run = session.scalars(cycles.select(cycles.Run)).one() + assert run.cycle_id == cycle_id diff --git a/radiator/tests/test_grid.py b/radiator/tests/test_grid.py new file mode 100644 index 0000000..dc5586d --- /dev/null +++ b/radiator/tests/test_grid.py @@ -0,0 +1,153 @@ +"""The acceptance run grid: trends, layout, and where it's shown.""" + +from datetime import datetime, timedelta, timezone + +import radiator_schema as schema +from radiator import grid, queries +from radiator.ingest import ingest_payload + +from . import factories as f + +T0 = datetime(2026, 9, 1, 6, tzinfo=timezone.utc) + + +def _run(started, env="ci", passed=8, failed=2, skipped=0, finished=True, **kw): + results = ( + [f.asset(f"P{i}") for i in range(passed)] + + [ + f.asset(f"F{i}", status="FAILED", agents=[f.agent("ars", "FAILED")]) + for i in range(failed) + ] + + [ + f.asset(f"S{i}", status="SKIPPED", agents=[f.agent("ars", "SKIPPED")]) + for i in range(skipped) + ] + ) + return schema.RunPayload( + run=f.run_create(started_at=started, env=env, **kw), + results=results, + finish=f.finish(started) if finished else None, + ) + + +def _load(app, *payloads): + with app.state.sessionmaker() as session: + for payload in payloads: + ingest_payload(session, payload) + session.commit() + + +def _items(app, **kw): + with app.state.sessionmaker() as session: + return grid.grid_runs( + session, + "sprint_4_tests", + kw.pop("since", T0 - timedelta(days=1)), + kw.pop("until", T0 + timedelta(days=60)), + **kw, + ) + + +def test_trends_compare_pass_rates_within_an_environment(app): + _load( + app, + _run(T0, passed=8, failed=2), # 80%: BASE + _run(T0 + timedelta(days=1), passed=9, failed=1), # 90%: UP + # a smaller run at the same rate is FLAT, not DOWN + _run(T0 + timedelta(days=2), passed=18, failed=2), + _run(T0 + timedelta(days=3), passed=7, failed=3), # 70%: DOWN + _run(T0 + timedelta(days=4), passed=0, failed=0, skipped=10), # no data + # compared with the last run that had a pass rate (70%) + _run(T0 + timedelta(days=5), passed=7, failed=3), + _run(T0 + timedelta(days=1), env="test", passed=5, failed=5), + _run(T0 + timedelta(days=2), env="dev", passed=1, failed=9), + ) + items = _items(app) + ci = [ + (i.trend, round(i.rate, 2) if i.rate is not None else None) + for i in items + if i.run.env == "ci" + ] + assert ci == [ + ("base", 0.8), + ("up", 0.9), + ("flat", 0.9), + ("down", 0.7), + ("nodata", None), + ("flat", 0.7), + ] + up = next(i for i in items if i.trend == "up") + assert round(up.delta, 1) == 10.0 + assert [i.trend for i in items if i.run.env == "test"] == ["base"] + assert [i.trend for i in items if i.run.env == "dev"] == ["excl"] + + +def test_only_finished_full_runs_are_shown(app): + _load( + app, + _run(T0), + _run(T0 + timedelta(hours=1), target="aragorn", target_url="http://x"), + _run(T0 + timedelta(hours=2), query_type="MVP1"), + _run(T0 + timedelta(hours=3), finished=False), + _run(T0 + timedelta(hours=4), suite="other_suite"), + ) + assert len(_items(app)) == 1 + + +def test_a_run_before_the_window_is_the_baseline(app): + _load( + app, + _run(T0, passed=5, failed=5), + _run(T0 + timedelta(days=20), passed=6, failed=4), + ) + [item] = _items(app, since=T0 + timedelta(days=10)) + assert item.trend == "up" and round(item.delta, 1) == 10.0 + + +def _item(started, env="ci"): + class R: + pass + + run = R() + run.started_at, run.env, run.number, run.id = started, env, 1, None + return grid.GridRun(run, 1, 0, 0, 1.0) + + +def test_layout_stacks_runs_too_close_to_sit_side_by_side(): + since, until = T0, T0 + timedelta(days=10) + items = [ + _item(T0 + timedelta(days=1)), + _item(T0 + timedelta(days=1, hours=6)), # overlaps the first: stacks + _item(T0 + timedelta(days=8)), # clear of both: back on top + _item(T0 + timedelta(days=2), env="prod"), + ] + g = grid.layout("s", items, since, until, width=1180) + assert [lane.env for lane in g.lanes] == ["ci", "prod"] + assert [c.level for c in g.lanes[0].cards] == [0, 1, 0] + # each run joined to the one before it in its lane + assert len(g.links) == 2 + # lanes don't overlap, and the busier one is taller + assert g.lanes[1].y == g.lanes[0].height > g.lanes[1].height + + +def test_grid_page(app, browser): + payloads = [_run(T0 + timedelta(days=d)) for d in range(3)] + _load(app, *payloads) + last = payloads[-1].run.run_id + + page = browser.get("/grid", params={"days": 90}).text + assert "Acceptance run grid" in page + assert f'href="/runs/{last}"' in page + assert "BASE" in page and "FLAT" in page + + +def test_runs_are_numbered_in_arrival_order_and_keep_their_number(app, browser): + first, second = _run(T0), _run(T0 + timedelta(days=1)) + _load(app, first, second) + _load(app, first) # re-uploaded: same number + with app.state.sessionmaker() as session: + a = queries.get_run(session, first.run.run_id) + b = queries.get_run(session, second.run.run_id) + assert b.number == a.number + 1 + assert f"#{b.number}" in browser.get(f"/runs/{second.run.run_id}").text + assert f"#{a.number}" in browser.get("/").text diff --git a/radiator/tests/test_ingest.py b/radiator/tests/test_ingest.py new file mode 100644 index 0000000..8e01c77 --- /dev/null +++ b/radiator/tests/test_ingest.py @@ -0,0 +1,184 @@ +"""The ingest API, including against the harness's own client.""" + +import uuid + +from fastapi.testclient import TestClient + +import radiator_schema as schema +from radiator import queries +from test_harness.radiator_client import RadiatorClient, push + +from . import factories as f +from .conftest import TOKEN + + +def _get(app, run_id): + with app.state.sessionmaker() as session: + return queries.get_run(session, run_id) + + +def test_full_run(app, api): + run = f.run_create() + assert ( + api.post("/api/ingest/runs", json=run.model_dump(mode="json")).status_code + == 201 + ) + batch = schema.ResultBatch( + results=[ + f.asset( + "Asset_1", + agents=[ + f.agent( + "ars", found=True, rank=3, score=0.9, n_results=120, pk="pk1" + ), + f.agent("aragorn", "NO_RESULTS", n_results=0, http_status=200), + ], + ) + ], + performance=[ + schema.PerformanceResult( + test_case_id="Perf_1", + asset_id="Asset_9", + host="https://ars", + status="PASSED", + max_sustainable_concurrency=12.5, + summary={"knee": {"users": 13}}, + ) + ], + ) + res = api.post( + f"/api/ingest/runs/{run.run_id}/results", json=batch.model_dump(mode="json") + ) + assert res.json() == {"results": 1, "performance": 1} + finish = f.finish(run.started_at, counts={"PASSED": 1}) + assert ( + api.patch( + f"/api/ingest/runs/{run.run_id}", json=finish.model_dump(mode="json") + ).status_code + == 200 + ) + + stored = _get(app, run.run_id) + assert stored.ended_at == finish.ended_at + assert stored.counts == {"PASSED": 1} + [asset] = stored.assets + ars, aragorn = asset.agents + assert (ars.found, ars.rank, ars.score, ars.n_results, ars.pk) == ( + True, + 3, + 0.9, + 120, + "pk1", + ) + assert (aragorn.status, aragorn.n_results) == ("NO_RESULTS", 0) + [perf] = stored.performance + assert perf.max_sustainable_concurrency == 12.5 + assert perf.summary == {"knee": {"users": 13}} + + +def test_writes_are_idempotent(app, api): + """Re-posting a run, or a result, updates it instead of duplicating it, + and an asset's agent rows are replaced by the newest upload.""" + run = f.run_create() + for _ in range(2): + api.post("/api/ingest/runs", json=run.model_dump(mode="json")) + first = schema.ResultBatch( + results=[f.asset("Asset_1", agents=[f.agent("ars"), f.agent("arax")])] + ) + second = schema.ResultBatch( + results=[f.asset("Asset_1", status="FAILED", agents=[f.agent("ars", "FAILED")])] + ) + for batch in (first, second): + api.post( + f"/api/ingest/runs/{run.run_id}/results", json=batch.model_dump(mode="json") + ) + + with app.state.sessionmaker() as session: + assert len(queries.list_runs(session)) == 1 + [asset] = _get(app, run.run_id).assets + assert asset.status == "FAILED" + assert [(a.agent, a.status) for a in asset.agents] == [("ars", "FAILED")] + + +def test_results_for_an_unknown_run_are_refused(api): + run_id = uuid.uuid4() + res = api.post(f"/api/ingest/runs/{run_id}/results", json={"results": []}) + assert res.status_code == 404 + res = api.patch( + f"/api/ingest/runs/{run_id}", + json=f.finish().model_dump(mode="json"), + ) + assert res.status_code == 404 + + +def test_invalid_body_is_rejected(api): + run = f.run_create() + api.post("/api/ingest/runs", json=run.model_dump(mode="json")) + res = api.post( + f"/api/ingest/runs/{run.run_id}/results", + json={ + "results": [ + { + "test_case_id": "x", + "asset_id": "y", + "kind": "acceptance", + "status": "MAYBE", + } + ] + }, + ) + assert res.status_code == 422 + + +def _harness_client(app, batch_size=2): + """The harness's RadiatorClient, talking to this app in-process.""" + client = RadiatorClient( + base_url="http://radiator", token=TOKEN, batch_size=batch_size + ) + client._client = TestClient( + app, + base_url=f"http://radiator{schema.INGEST_PREFIX}", + headers={"Authorization": f"Bearer {TOKEN}"}, + ) + return client + + +def test_harness_client_round_trip(app): + """The contract test: what the harness sends is what the radiator stores.""" + client = _harness_client(app) + run = f.run_create() + client.start_run(run) + for i in range(5): + client.add_result(f.asset(f"Asset_{i}")) + client.finish_run(counts={"PASSED": 5}) + assert not client.upload_failed + + stored = _get(app, run.run_id) + assert sorted(a.asset_id for a in stored.assets) == [f"Asset_{i}" for i in range(5)] + assert stored.counts == {"PASSED": 5} + + # and it can read back what the Slack report uses + client._client = TestClient(app, headers={"Authorization": f"Bearer {TOKEN}"}) + assert client.summary()["pass_rate"] == 1.0 + assert client.history_png().startswith(b"\x89PNG") + + +def test_replaying_a_partly_uploaded_run(app): + """A run that half made it, replayed from the saved payload, ends up whole + and without duplicates.""" + run = f.run_create() + partial = _harness_client(app) + partial.start_run(run) + partial.add_result(f.asset("Asset_0")) + partial.add_result(f.asset("Asset_1")) # flushed: batch of 2 + + payload = schema.RunPayload( + run=run, + results=[f.asset(f"Asset_{i}") for i in range(4)], + finish=f.finish(run.started_at), + ) + assert push(payload, _harness_client(app)) + + stored = _get(app, run.run_id) + assert len(stored.assets) == 4 + assert stored.ended_at == payload.finish.ended_at diff --git a/radiator/tests/test_live.py b/radiator/tests/test_live.py new file mode 100644 index 0000000..c34040a --- /dev/null +++ b/radiator/tests/test_live.py @@ -0,0 +1,154 @@ +"""Live updates: progress, what's running, and the pages that poll for it.""" + +import uuid +from datetime import datetime, timedelta, timezone + +from sqlalchemy import update + +import radiator_schema as schema +from radiator.models import Run + +from . import factories as f + + +def _open(api, expected_assets=3, **kw): + run = f.run_create(expected_assets=expected_assets, **kw) + assert ( + api.post("/api/ingest/runs", json=run.model_dump(mode="json")).status_code + == 201 + ) + return run.run_id + + +def _upload(api, run_id, *asset_ids): + batch = schema.ResultBatch(results=[f.asset(a) for a in asset_ids]) + res = api.post( + f"/api/ingest/runs/{run_id}/results", json=batch.model_dump(mode="json") + ) + assert res.status_code == 200 + + +def _finish(api, run_id): + finish = f.finish().model_dump(mode="json") + assert api.patch(f"/api/ingest/runs/{run_id}", json=finish).status_code == 200 + + +def test_progress_follows_a_run_until_it_finishes(api): + run_id = _open(api) + first = api.get(f"/api/runs/{run_id}/progress").json() + assert first["running"] and first["state"] == "running" + assert (first["done_assets"], first["expected_assets"]) == (0, 3) + + _upload(api, run_id, "A1", "A2") + second = api.get(f"/api/runs/{run_id}/progress").json() + assert second["done_assets"] == 2 + assert second["counts"] == {"PASSED": 2} + assert second["version"] != first["version"] + + _finish(api, run_id) + done = api.get(f"/api/runs/{run_id}/progress").json() + assert not done["running"] and done["state"] == "finished" + assert done["version"] != second["version"] + + +def test_a_run_that_stopped_uploading_did_not_finish(app, api): + run_id = _open(api) + with app.state.sessionmaker() as session: + session.execute( + update(Run) + .where(Run.id == run_id) + .values(updated_at=datetime.now(timezone.utc) - timedelta(hours=3)) + ) + session.commit() + progress = api.get(f"/api/runs/{run_id}/progress").json() + assert progress["state"] == "stalled" and not progress["running"] + assert api.get("/api/live").json()["running"] == [] + + +def test_live_status_changes_when_runs_start_upload_and_finish(api): + empty = api.get("/api/live").json() + run_id = _open(api) + started = api.get("/api/live").json() + assert started["running"] == [str(run_id)] + assert started["version"] != empty["version"] + _upload(api, run_id, "A1") + uploaded = api.get("/api/live").json() + assert uploaded["version"] != started["version"] + _finish(api, run_id) + finished = api.get("/api/live").json() + assert finished["running"] == [] + assert finished["version"] != uploaded["version"] + + +def test_a_running_runs_page_polls_and_shows_progress(api, browser): + run_id = _open(api) + _upload(api, run_id, "A1") + page = browser.get(f"/runs/{run_id}").text + assert f'data-live="/api/runs/{run_id}/progress"' in page + assert "data-live-until-done" in page + assert "1 of 3 assets" in page and 'aria-valuenow="1"' in page + for region in ( + "run-meta", + "run-progress", + "run-tiles", + "run-agents", + "run-results", + ): + assert f'id="{region}" data-live-region' in page, region + assert "running" in page + + _finish(api, run_id) + page = browser.get(f"/runs/{run_id}").text + assert "data-live=" not in page + assert "of 3 assets" not in page + + +def test_a_run_with_no_results_yet_says_so(api, browser): + run_id = _open(api, expected_assets=None) + page = browser.get(f"/runs/{run_id}").text + assert "0 assets so far" in page + assert "No results yet" in page + + +def test_the_runs_list_and_weekly_page_poll(app, api, browser): + running = _open(api) + stalled = _open(api, started_at=datetime.now(timezone.utc) - timedelta(hours=5)) + with app.state.sessionmaker() as session: + session.execute( + update(Run) + .where(Run.id == stalled) + .values(updated_at=datetime.now(timezone.utc) - timedelta(hours=4)) + ) + session.commit() + version = api.get("/api/live").json()["version"] + runs = browser.get("/").text + assert 'data-live="/api/live"' in runs + assert f'data-live-version="{version}"' in runs + assert 'id="runs-table" data-live-region' in runs + assert runs.count("running") == 1 + assert "didn't finish" in runs or "didn't finish" in runs + weekly = browser.get("/weekly").text + assert 'data-live="/api/live"' in weekly + # a past week doesn't change, so it doesn't poll + last_month = (datetime.now(timezone.utc) - timedelta(days=30)).date().isoformat() + assert ( + "data-live=" not in browser.get("/weekly", params={"week_of": last_month}).text + ) + assert running != stalled + + +def test_live_endpoints_need_a_login_or_token(client): + assert client.get("/api/live").status_code == 401 + assert client.get(f"/api/runs/{uuid.uuid4()}/progress").status_code == 401 + + +def test_what_changed_mid_run_leaves_out_untested_assets(api, browser): + before = _open(api) + _upload(api, before, "A1", "A2", "A3") + _finish(api, before) + run_id = _open(api) + _upload(api, run_id, "A1") + page = browser.get(f"/runs/{run_id}/diff").text + assert "This run hasn't finished" in page or "This run hasn't finished" in page + assert 'href="#removed">0' in page + assert "Pass rate so far" in browser.get(f"/runs/{run_id}").text diff --git a/radiator/tests/test_logging.py b/radiator/tests/test_logging.py new file mode 100644 index 0000000..14e5e64 --- /dev/null +++ b/radiator/tests/test_logging.py @@ -0,0 +1,79 @@ +"""The radiator's log: every upload, and why one was rejected.""" + +import logging +import uuid + +import pytest + +import radiator_schema as schema + +from . import factories as f + + +class _Records(logging.Handler): + def __init__(self): + super().__init__(logging.INFO) + self.records = [] + + def emit(self, record): + self.records.append(record) + + @property + def text(self): + return "\n".join(r.getMessage() for r in self.records) + + +@pytest.fixture +def radiator_log(): + # the radiator's logger doesn't propagate (it has its own handler) + logger = logging.getLogger("radiator") + handler = _Records() + logger.addHandler(handler) + previous = logger.level + logger.setLevel(logging.INFO) + yield handler + logger.removeHandler(handler) + logger.setLevel(previous) + + +def test_uploads_are_logged(api, radiator_log): + run = f.run_create(expected_assets=2, target="aragorn") + api.post("/api/ingest/runs", json=run.model_dump(mode="json")) + batch = schema.ResultBatch(results=[f.asset("A1"), f.asset("A2")]) + api.post( + f"/api/ingest/runs/{run.run_id}/results", json=batch.model_dump(mode="json") + ) + api.patch(f"/api/ingest/runs/{run.run_id}", json=f.finish().model_dump(mode="json")) + lines = [r.getMessage() for r in radiator_log.records] + assert lines == [ + f"Run {run.run_id} opened: sprint_4_tests in ci, target aragorn, " + "2 assets expected", + f"Run {run.run_id}: 2 asset results, 0 performance results", + f"Run {run.run_id} finished", + ] + + +def test_a_rejected_upload_says_what_was_wrong(api, radiator_log): + run = f.run_create() + api.post("/api/ingest/runs", json=run.model_dump(mode="json")) + bad = {"results": [{"test_case_id": "TC", "asset_id": "A1", "status": "MAYBE"}]} + res = api.post(f"/api/ingest/runs/{run.run_id}/results", json=bad) + assert res.status_code == 422 + warning = radiator_log.records[-1].getMessage() + assert warning.startswith( + f"Rejected upload POST /api/ingest/runs/{run.run_id}/results: " + ) + assert "body.results.0.status" in warning and "body.results.0.kind" in warning + + +def test_results_for_an_unknown_run_are_logged(api, radiator_log): + run_id = uuid.uuid4() + res = api.post(f"/api/ingest/runs/{run_id}/results", json={"results": []}) + assert res.status_code == 404 + assert f"Upload for unknown run {run_id}" in radiator_log.text + + +def test_the_read_api_and_pages_dont_log_validation_noise(api, browser, radiator_log): + browser.get("/runs/not-a-uuid") + api.get("/api/runs", params={"limit": "lots"}) + assert "Rejected upload" not in radiator_log.text diff --git a/radiator/tests/test_pages.py b/radiator/tests/test_pages.py new file mode 100644 index 0000000..ce89f9b --- /dev/null +++ b/radiator/tests/test_pages.py @@ -0,0 +1,535 @@ +"""The dashboard's pages and the queries behind them.""" + +from datetime import datetime, timedelta, timezone + +import radiator_schema as schema +from radiator import queries +from radiator.ingest import ingest_payload + +from . import factories as f + +T0 = datetime(2026, 9, 1, 6, tzinfo=timezone.utc) + + +def _load(app, *payloads): + with app.state.sessionmaker() as session: + for payload in payloads: + ingest_payload(session, payload) + session.commit() + + +def _payload(started_at, results, **run_kwargs): + run = f.run_create(started_at=started_at, **run_kwargs) + return schema.RunPayload(run=run, results=results, finish=f.finish(started_at)) + + +def test_pages_render(app, browser): + perf = schema.RunPayload( + run=f.run_create(started_at=T0, suite="performance_tests"), + performance=[ + schema.PerformanceResult( + test_case_id="P", + asset_id="A", + host="https://ars.ci.transltr.io", + helmsdeep_target="ars", + status="PASSED", + max_sustainable_concurrency=12.0, + ) + ], + ) + first = _payload(T0, [f.asset("Asset_1", agents=[f.agent("ars", rank=4)])]) + second = _payload( + T0 + timedelta(days=1), + [ + f.asset( + "Asset_1", status="FAILED", agents=[f.agent("ars", "FAILED", rank=90)] + ) + ], + ) + _load(app, perf, first, second) + run_id = second.run.run_id + + runs = browser.get("/") + assert runs.status_code == 200 + assert "sprint_4_tests" in runs.text and "performance_tests" in runs.text + + page = browser.get(f"/runs/{run_id}") + assert page.status_code == 200 + assert "Asset_1 name" in page.text + assert 'href="/runs/{}/diff#regression">1<'.format(run_id) in page.text + + diff = browser.get(f"/runs/{run_id}/diff") + assert diff.status_code == 200 + assert "4 → 90" in diff.text + + asset = browser.get("/assets/TestCase_1/Asset_1?env=ci") + assert asset.status_code == 200 + # a run without an env links with env= empty: still one env's history + assert "last 2 runs in ci" in browser.get("/assets/TestCase_1/Asset_1?env=").text + assert "Rank of the expected answer" in asset.text + + assert browser.get("/trends").status_code == 200 + performance = browser.get("/performance") + assert performance.status_code == 200 + assert "https://ars.ci.transltr.io (ars, default, ci)" in performance.text + + +def test_empty_database(browser): + assert "No runs yet" in browser.get("/").text + assert browser.get("/trends").status_code == 200 + assert browser.get("/performance").status_code == 200 + + +def test_missing_things_404(browser): + assert browser.get("/runs/6e3c1d8e-5f0e-4bb8-9d7e-1f1e0b6f0a11").status_code == 404 + assert browser.get("/assets/nope/nope").status_code == 404 + + +def test_uploaded_text_is_escaped(app, browser): + """Names and messages come from uploads; they must never become markup.""" + evil = '' + payload = _payload( + T0, + [ + f.asset( + "Asset_1", name=evil, agents=[f.agent("ars", "FAILED", message=evil)] + ) + ], + ) + _load(app, payload) + page = browser.get(f"/runs/{payload.run.run_id}").text + assert evil not in page + assert "<script>" in page + + +def test_matrix_filters(app, browser): + payload = _payload( + T0, + [ + f.asset("Asset_1", agents=[f.agent("ars"), f.agent("arax", "FAILED")]), + f.asset("Asset_2", status="FAILED", agents=[f.agent("ars", "FAILED")]), + ], + ) + _load(app, payload) + url = f"/runs/{payload.run.run_id}" + failed_overall = browser.get(url, params={"status": "FAILED"}).text + assert "Asset_2 name" in failed_overall and "Asset_1 name" not in failed_overall + failed_arax = browser.get(url, params={"status": "FAILED", "agent": "arax"}).text + assert "Asset_1 name" in failed_arax and "Asset_2 name" not in failed_arax + search = browser.get(url, params={"q": "asset_2"}).text + assert "Asset_2 name" in search and "Asset_1 name" not in search + + +def _run(app, payload): + with app.state.sessionmaker() as session: + return queries.get_run(session, payload.run.run_id) + + +def test_diff_classification(app): + before = _payload( + T0, + [ + f.asset("regressed", agents=[f.agent("ars", rank=3)]), + f.asset("fixed", agents=[f.agent("ars", "FAILED")]), + f.asset("changed", agents=[f.agent("ars", "FAILED")]), + f.asset("skipped", agents=[f.agent("ars")]), + f.asset("moved", agents=[f.agent("ars", rank=40)]), + f.asset("jittered", agents=[f.agent("ars", rank=10)]), + f.asset("dropped"), + ], + ) + after = _payload( + T0 + timedelta(days=1), + [ + f.asset("regressed", agents=[f.agent("ars", "FAILED", rank=80)]), + f.asset("fixed", agents=[f.agent("ars")]), + f.asset("changed", agents=[f.agent("ars", "NO_RESULTS")]), + f.asset("skipped", agents=[f.agent("ars", "SKIPPED")]), + f.asset("moved", agents=[f.agent("ars", rank=4)]), + f.asset("jittered", agents=[f.agent("ars", rank=12)]), + f.asset("new"), + ], + ) + _load(app, before, after) + diff = queries.diff_runs(_run(app, before), _run(app, after)) + found = { + kind: [c.asset.asset_id for c in changes] + for kind, changes in diff.changes.items() + } + assert found == { + "regression": ["regressed"], + "fixed": ["fixed"], + "changed": ["changed"], + "skipped": ["skipped"], + # a 2-place wobble isn't a move + "rank": ["moved"], + "added": ["new"], + "removed": ["dropped"], + } + [moved] = diff.changes["rank"] + assert moved.rank_delta == 36 + + +def test_previous_run_stays_in_its_series(app): + """A run with a target override or another query type isn't the + 'previous run' of a full run, and doesn't show up in its trends.""" + full_1 = _payload(T0, [f.asset()]) + override = _payload( + T0 + timedelta(hours=1), + [f.asset()], + target="aragorn", + target_url="http://localhost:8080", + ) + mvp1 = _payload(T0 + timedelta(hours=2), [f.asset()], query_type="MVP1") + other_env = _payload(T0 + timedelta(hours=3), [f.asset()], env="test") + full_2 = _payload(T0 + timedelta(hours=4), [f.asset()]) + _load(app, full_1, override, mvp1, other_env, full_2) + + with app.state.sessionmaker() as session: + latest = queries.get_run(session, full_2.run.run_id) + assert queries.previous_run(session, latest).id == full_1.run.run_id + assert ( + queries.previous_run(session, queries.get_run(session, full_1.run.run_id)) + is None + ) + points = queries.agent_trends(session, "sprint_4_tests", "ci", days=100000) + assert [p.run.id for p in points] == [full_1.run.run_id, full_2.run.run_id] + + +def test_pass_rate_leaves_out_skips(): + assert queries.pass_rate({"PASSED": 3, "FAILED": 1, "SKIPPED": 10}) == 0.75 + assert queries.pass_rate({"SKIPPED": 4}) is None + + +def test_an_unfinished_run_is_not_a_baseline(app): + finished = _payload(T0, [f.asset()]) + crashed = schema.RunPayload( + run=f.run_create(started_at=T0 + timedelta(hours=1)), results=[f.asset()] + ) + latest = _payload(T0 + timedelta(hours=2), [f.asset()]) + _load(app, finished, crashed, latest) + with app.state.sessionmaker() as session: + run = queries.get_run(session, latest.run.run_id) + assert queries.previous_run(session, run).id == finished.run.run_id + trend_runs = [ + p.run.id + for p in queries.agent_trends(session, "sprint_4_tests", "ci", 100000) + ] + assert crashed.run.run_id not in trend_runs + + +def test_asset_page_environments(app, browser): + """The default is the env with the most history; runs with no env have + their own history (eg imported ones).""" + _load( + app, + _payload(T0, [f.asset()], env="test"), + _payload(T0 + timedelta(hours=1), [f.asset()], env="test"), + _payload(T0 + timedelta(hours=2), [f.asset()], env="prod"), + _payload(T0 + timedelta(hours=3), [f.asset()], env=None), + ) + assert "last 2 runs in test" in browser.get("/assets/TestCase_1/Asset_1").text + no_env = browser.get("/assets/TestCase_1/Asset_1?env=-").text + assert "last 1 runs with no recorded environment" in no_env + + +def test_run_summary_and_history_png(app, api, client): + first = _payload( + T0, + [ + f.asset("A1"), + f.asset("A2", status="FAILED", agents=[f.agent("ars", "FAILED")]), + ], + ) + second = _payload( + T0 + timedelta(days=1), + [ + f.asset("A1", status="FAILED", agents=[f.agent("ars", "FAILED")]), + f.asset("A2"), + ], + ) + other_series = _payload(T0 + timedelta(hours=12), [f.asset("A1")], env="test") + later = _payload(T0 + timedelta(days=2), [f.asset("A1")]) + _load(app, first, second, other_series, later) + run_id = second.run.run_id + + summary = api.get(f"/api/runs/{run_id}/summary").json() + assert summary["pass_rate"] == 0.5 + assert summary["previous_run_id"] == str(first.run.run_id) + assert summary["previous_pass_rate"] == 0.5 + assert (summary["regressions"], summary["fixed"]) == (1, 1) + + res = api.get(f"/api/runs/{run_id}/history.png") + assert res.status_code == 200 + assert res.headers["content-type"] == "image/png" + assert res.content.startswith(b"\x89PNG") + assert client.get(f"/api/runs/{run_id}/history.png").status_code == 401 + + with app.state.sessionmaker() as session: + run = queries.get_run(session, run_id) + history = queries.series_history(session, run) + # same series only, and nothing after this run + assert [r.id for r, _ in history] == [first.run.run_id, run_id] + assert history[-1][1]["ars"] == {"FAILED": 1, "PASSED": 1} + + +def test_summary_of_a_first_run(app, api): + only = _payload(T0, [f.asset()]) + _load(app, only) + summary = api.get(f"/api/runs/{only.run.run_id}/summary").json() + assert summary["pass_rate"] == 1.0 + assert summary["previous_run_id"] is None and summary["regressions"] is None + assert api.get(f"/api/runs/{only.run.run_id}/history.png").content.startswith( + b"\x89PNG" + ) + + +def _perf_run(started, target="arax", msc=7.0, profile="mixed", env="ci", passed=True): + return schema.RunPayload( + run=f.run_create( + started_at=started, + suite="performance_tests", + env=env, + target=target, + target_url=f"https://{target}", + ), + performance=[ + schema.PerformanceResult( + test_case_id="Perf_1", + asset_id="Perf_mixed", + host=f"https://{target}", + helmsdeep_target=target, + profile=profile, + status="PASSED" if passed else "FAILED", + max_sustainable_concurrency=msc, + checkpoints_passed=passed, + ) + ], + finish=f.finish(started), + ) + + +def test_performance_history_for_a_run(app, api): + """A service's series: same host, run type, profile and env; other + services, profiles and envs, and later runs, stay out of it.""" + earlier = _perf_run(T0, msc=6.0, passed=False) + other_service = _perf_run(T0 + timedelta(hours=1), target="ars", msc=14.0) + other_profile = _perf_run(T0 + timedelta(hours=2), msc=3.0, profile=None) + other_env = _perf_run(T0 + timedelta(hours=3), msc=2.0, env="test") + this = _perf_run(T0 + timedelta(days=1), msc=7.5) + later = _perf_run(T0 + timedelta(days=2), msc=9.0) + _load(app, earlier, other_service, other_profile, other_env, this, later) + + with app.state.sessionmaker() as session: + run = queries.get_run(session, this.run.run_id) + [(perf, history)] = queries.performance_series(session, run) + assert [p.max_sustainable_concurrency for _, p in history] == [6.0, 7.5] + + summary = api.get(f"/api/runs/{this.run.run_id}/summary").json() + [entry] = summary["performance"] + assert entry["max_sustainable_concurrency"] == 7.5 + assert entry["previous_max_sustainable_concurrency"] == 6.0 + assert summary["pass_rate"] is None + + res = api.get(f"/api/runs/{this.run.run_id}/performance.png") + assert res.status_code == 200 and res.content.startswith(b"\x89PNG") + + +def test_performance_png_needs_performance_results(app, api): + acceptance = _payload(T0, [f.asset()]) + _load(app, acceptance) + assert ( + api.get(f"/api/runs/{acceptance.run.run_id}/performance.png").status_code == 404 + ) + + +def test_dashboard_lines_are_steps(): + """Each run's value holds until the next: every segment of the line is + horizontal or vertical.""" + import re + from datetime import datetime, timezone + + from radiator import charts + + svg = charts.line_chart( + [ + charts.Series( + "ars", + "red", + [ + (datetime(2026, 9, d, tzinfo=timezone.utc), v) + for d, v in [(1, 0.5), (2, 0.9), (3, 0.7)] + ], + ) + ] + ) + [points] = re.findall(r'(Asset_\d{3}) name<", html) + + +def test_results_are_paginated(app, browser): + url = _big_run(app) + + first = browser.get(url).text + assert _shown(first) == [f"Asset_{i:03d}" for i in range(50)] + assert "Showing 1–50 of 120 assets" in first + assert "Page 1 of 3" in first + + last = browser.get(url, params={"page": 3}).text + assert _shown(last) == [f"Asset_{i:03d}" for i in range(100, 120)] + assert "Showing 101–120 of 120 assets" in last + + # past the end: the last page, not an empty one + assert _shown(browser.get(url, params={"page": 99}).text) == _shown(last) + + bigger = browser.get(url, params={"per_page": 100}).text + assert len(_shown(bigger)) == 100 and "Page 1 of 2" in bigger + everything = browser.get(url, params={"per_page": "all"}).text + assert len(_shown(everything)) == 120 and "Page 1 of" not in everything + # an unknown page size falls back to the default + assert len(_shown(browser.get(url, params={"per_page": 7}).text)) == 50 + + +def test_pages_keep_their_filters(app, browser): + url = _big_run(app, n=600) + page = browser.get(url, params={"status": "FAILED", "per_page": 50}).text + # 60 failures over two pages, and the links carry the filter + assert "Showing 1–50 of 60 assets matching" in page + assert len(_shown(page)) == 50 + assert "status=FAILED&per_page=50&page=2#matrix" in page + second = browser.get( + url, params={"status": "FAILED", "per_page": 50, "page": 2} + ).text + assert _shown(second) == [f"Asset_{i:03d}" for i in range(500, 600, 10)] + # the filter form keeps the page size, and drops the page + assert 'type="hidden" name="per_page" value="50"' in page + + +def test_runs_list_is_paginated(app, browser): + import re + + _load( + app, + *[ + _payload( + T0 + timedelta(hours=i), [f.asset()], env="test" if i % 3 == 0 else "ci" + ) + for i in range(60) + ], + ) + + def started(html): + return re.findall(r">(2026-09-\d\d \d\d:\d\d) UTC", html) + + first = browser.get("/").text + assert "Showing 1–25 of 60 runs, newest first" in first + assert "Page 1 of 3" in first + # newest first + assert started(first)[0] == "2026-09-03 17:00" + assert len(started(first)) == 25 + + last = browser.get("/", params={"page": 3}).text + assert "Showing 51–60 of 60 runs" in last + assert started(last)[-1] == "2026-09-01 06:00" + + assert len(started(browser.get("/", params={"per_page": 50}).text)) == 50 + # no "all" for runs: it falls back to the default + assert len(started(browser.get("/", params={"per_page": "all"}).text)) == 25 + + filtered = browser.get("/", params={"env": "test", "per_page": 50}).text + assert "Showing 1–20 of 20 runs" in filtered + assert 'type="hidden" name="per_page" value="50"' in filtered + # one page: no pager + assert "Page 1 of" not in filtered + + paged = browser.get("/", params={"env": "ci"}).text + assert "env=ci&page=2" in paged + + +def test_breakdown_by_agent_and_expected_output(app, browser): + payload = _payload( + T0, + [ + f.asset( + "A1", + expected_output="TopAnswer", + agents=[f.agent("ars"), f.agent("arax", "FAILED")], + ), + f.asset( + "A2", + expected_output="TopAnswer", + status="FAILED", + agents=[f.agent("ars", "FAILED"), f.agent("arax")], + ), + f.asset( + "A3", + expected_output="NeverShow", + agents=[f.agent("ars"), f.agent("arax", "NO_RESULTS")], + ), + f.asset("A4", expected_output="Acceptable", agents=[f.agent("ars")]), + ], + ) + _load(app, payload) + + with app.state.sessionmaker() as session: + run = queries.get_run(session, payload.run.run_id) + counts = queries.agent_expected_counts(run) + # expected outputs in their fixed order, per agent + assert counts["ars"] == [ + ("TopAnswer", {"PASSED": 1, "FAILED": 1}), + ("Acceptable", {"PASSED": 1}), + ("NeverShow", {"PASSED": 1}), + ] + assert counts["arax"] == [ + ("TopAnswer", {"FAILED": 1, "PASSED": 1}), + ("NeverShow", {"NO_RESULTS": 1}), + ] + + page = browser.get(f"/runs/{payload.run.run_id}").text + assert "By agent and expected output" in page + # a count opens the grid filtered to that agent, status, and expected output + assert "agent=arax&status=FAILED&expected=TopAnswer#matrix" in page + filtered = browser.get( + f"/runs/{payload.run.run_id}", + params={"agent": "arax", "status": "FAILED", "expected": "TopAnswer"}, + ).text + assert "A1 name" in filtered and "A2 name" not in filtered diff --git a/radiator/tests/test_trapi_version.py b/radiator/tests/test_trapi_version.py new file mode 100644 index 0000000..b781389 --- /dev/null +++ b/radiator/tests/test_trapi_version.py @@ -0,0 +1,74 @@ +"""A run's TRAPI version, and a change of version between runs.""" + +import uuid +from datetime import datetime, timedelta, timezone + +import radiator_schema as schema +from radiator import cycles, queries +from radiator.ingest import ingest_payload + +from . import factories as f + +T0 = datetime(2026, 9, 7, 6, tzinfo=timezone.utc) + + +def _run(app, started, trapi, status="PASSED", cycle_id=None): + payload = schema.RunPayload( + run=f.run_create(started_at=started, trapi_version=trapi, cycle_id=cycle_id), + results=[f.asset("A1", status=status, agents=[f.agent("ars", status)])], + finish=f.finish(started), + ) + with app.state.sessionmaker() as session: + ingest_payload(session, payload) + session.commit() + return payload.run.run_id + + +def test_trapi_change(): + class R: + def __init__(self, v): + self.trapi_version = v + + assert queries.trapi_change(R("1.6.0"), R("2.0.0")) == ("1.6", "2.0") + assert queries.trapi_change(R("2.0.0"), R("2.0.1")) is None # patch only + assert queries.trapi_change(R(None), R("2.0.0")) is None # unknown (imported) + assert queries.trapi_change(None, R("2.0.0")) is None + + +def test_an_upgrade_is_called_out(app, api, browser): + _run(app, T0, "1.6.0") + cycle_id = uuid.uuid4() + after = _run( + app, T0 + timedelta(days=7), "2.0.0", status="FAILED", cycle_id=cycle_id + ) + + summary = api.get(f"/api/runs/{after}/summary").json() + assert (summary["previous_trapi_version"], summary["trapi_version"]) == ( + "1.6.0", + "2.0.0", + ) + assert ( + api.get(f"/api/runs/{after}", params={"results": "false"}).json()[ + "trapi_version" + ] + == "2.0.0" + ) + + page = browser.get(f"/runs/{after}").text + assert "TRAPI 2.0" in page + assert "TRAPI 1.6 → 2.0 since the previous run" in page + assert "TRAPI 1.6 → 2.0" in browser.get(f"/runs/{after}/diff").text + assert "TRAPI 2.0" in browser.get("/").text + + with app.state.sessionmaker() as session: + row = cycles.get_cycle(session, cycle_id).rows[0] + assert row.flags[0].kind == "trapi_change" + assert "TRAPI 1.6 → 2.0 since the previous run" in row.flags[0].text + # still compared: the regression shows, with the reason beside it + assert row.regressions == 1 + + +def test_the_same_version_says_nothing(app, browser): + _run(app, T0, "2.0.0") + after = _run(app, T0 + timedelta(days=7), "2.0.0") + assert "since the previous run: some" not in browser.get(f"/runs/{after}").text diff --git a/radiator/tests/test_zebrunner_import.py b/radiator/tests/test_zebrunner_import.py new file mode 100644 index 0000000..c79e11f --- /dev/null +++ b/radiator/tests/test_zebrunner_import.py @@ -0,0 +1,538 @@ +"""The Zebrunner import, against a fake Zebrunner speaking its read API.""" + +import json +from datetime import datetime, timezone + +import httpx +import pytest +from fastapi.testclient import TestClient + +from radiator import queries +from radiator.zebrunner_import import ( + ArsEnvResolver, + Radiator, + Zebrunner, + ZebrunnerError, + ars_hosts_from_smartapi, + convert_test, + infer_env, + import_runs, + report_pks, + run_id_for, + suite_from_name, +) + +from .conftest import TOKEN + +ZE = "http://zebrunner.test" + + +def label(key, value): + return {"key": key, "value": value} + + +def acceptance_test(test_id, asset, agents, status="PASSED", expected="TopAnswer"): + return { + "id": test_id, + "name": f"{asset} name", + "status": status, + "testClass": "TestCase_1", + "labels": [ + label("TestCase", "TestCase_1"), + label("TestAsset", asset), + label("ExpectedOutput", expected), + label("InputCurie", "MONDO:0004979"), + label("OutputCurie", "CHEBI:31690"), + *[label(agent, s) for agent, s in agents.items()], + ], + } + + +REPORT = { + "pks": {"parent_pk": "parent-1", "ars": "merged-1", "aragorn": "child-1"}, + "result": { + "ars": { + "status": "PASSED", + "message": None, + "actual_output": {"found": True, "ars_rank": 3, "ars_score": 0.97}, + }, + "aragorn": { + "status": "FAILED", + "message": None, + "actual_output": {"found": True, "ara_rank": 88, "ara_score": 0.2}, + }, + }, + "test_details": None, +} + +LAUNCHES = [ + { + "id": 101, + "name": "sprint_4_tests: 2025_03_02_06_00", + "startedAt": "2025-03-02T06:00:00Z", + "endedAt": "2025-03-02T06:50:00Z", + "build": "v0.3.3", + }, + { + "id": 100, + "name": "sprint_4_tests: 2025_03_01_06_00", + "startedAt": "2025-03-01T06:00:00Z", + "endedAt": "2025-03-01T06:45:00Z", + "build": "v0.3.3", + }, + { + "id": 99, + "name": "performance_tests: 2025_02_28_06_00", + "startedAt": "2025-02-28T06:00:00Z", + "endedAt": "2025-02-28T07:00:00Z", + }, +] +TESTS = { + 101: [ + acceptance_test(1, "Asset_1", {"ars": "PASSED", "aragorn": "FAILED"}), + # Zebrunner refused the NO_RESULTS finish, so the test is IN_PROGRESS; + # the agent label still has the harness's status + acceptance_test( + 2, + "Asset_2", + {"ars": "NO_RESULTS", "aragorn": "SKIPPED"}, + status="IN_PROGRESS", + ), + ], + 100: [ + acceptance_test( + 3, "Asset_1", {"ars": "FAILED", "aragorn": "FAILED"}, status="FAILED" + ) + ], + 99: [ + { + "id": 4, + "name": "perf", + "status": "PASSED", + "labels": [ + label("TestCase", "Perf_1"), + label("TestAsset", "Asset_9"), + label("TestRunTime", "60"), + label("UserSpawnRate", "1"), + ], + } + ], +} +LOGS = { + 1: [ + {"kind": "LOG", "message": json.dumps({"message": {"query_graph": {}}})}, + {"kind": "LOG", "message": json.dumps(REPORT, indent=4)}, + ], +} + + +def fake_zebrunner(calls): + """Zebrunner's read API, with an access token that expires once.""" + state = {"expired_once": False} + + def handler(request: httpx.Request): + calls.append((request.method, request.url.path, dict(request.url.params))) + path = request.url.path + if path == "/api/iam/v1/auth/refresh": + assert json.loads(request.content) == {"refreshToken": "refresh"} + return httpx.Response(200, json={"authToken": "access"}) + assert request.headers["Authorization"] == "Bearer access" + if path == "/api/reporting/v1/launches": + if not state["expired_once"]: + state["expired_once"] = True + return httpx.Response(401) + page, size = int(request.url.params["page"]), int( + request.url.params["pageSize"] + ) + items = LAUNCHES[(page - 1) * size : page * size] + return httpx.Response(200, json={"items": items, "_meta": {}}) + if path.startswith("/api/reporting/v1/launches/") and path.endswith("/tests"): + launch_id = int(path.split("/")[5]) + return httpx.Response(200, json={"items": TESTS[launch_id]}) + if path.endswith("/logs-and-screenshots"): + test_id = int(path.split("/")[7]) + logs = LOGS.get(test_id, []) + # one log per page, to exercise the page token + token = int(request.url.params.get("pageToken", 0)) + meta = {"nextPageToken": str(token + 1)} if token + 1 < len(logs) else {} + return httpx.Response( + 200, json={"items": logs[token : token + 1], "_meta": meta} + ) + return httpx.Response(404) + + return httpx.MockTransport(handler) + + +def _radiator(app): + radiator = Radiator.__new__(Radiator) + radiator.client = TestClient(app, headers={"Authorization": f"Bearer {TOKEN}"}) + return radiator + + +def _zebrunner(calls): + zebrunner = Zebrunner(ZE, "refresh", project_id=7, transport=fake_zebrunner(calls)) + return zebrunner + + +def test_import(app): + calls = [] + stats = import_runs(_zebrunner(calls), _radiator(app), env="ci") + + assert stats.imported == 2 + assert stats.skipped_empty == 1 # the performance-only run + assert stats.performance_tests_skipped == 1 + assert not stats.failed + # every Zebrunner call carried the project id, paging worked through the 401 + assert all( + params.get("projectId") == "7" + for _, path, params in calls + if "auth" not in path + ) + + with app.state.sessionmaker() as session: + latest = queries.get_run(session, run_id_for(ZE, 101)) + assert latest.origin == "zebrunner_import" + assert latest.origin_ref == "101" + assert latest.suite == "sprint_4_tests" + assert latest.env == "ci" + assert latest.harness_version == "v0.3.3" + assert latest.counts == {"PASSED": 1, "NO_RESULTS": 1} + + assets = {a.asset_id: a for a in latest.assets} + one = assets["Asset_1"] + assert one.status == "PASSED" + assert one.parent_pk == "parent-1" + agents = {a.agent: a for a in one.agents} + assert ( + agents["ars"].found, + agents["ars"].rank, + agents["ars"].score, + agents["ars"].pk, + ) == (True, 3, 0.97, "merged-1") + assert (agents["aragorn"].status, agents["aragorn"].rank) == ("FAILED", 88) + # what Zebrunner never had stays unknown, not zero + assert agents["ars"].n_results is None and agents["ars"].http_status is None + + two = assets["Asset_2"] + assert two.status == "NO_RESULTS" + assert {a.agent: a.status for a in two.agents} == { + "ars": "NO_RESULTS", + "aragorn": "SKIPPED", + } + + # the two imported runs are one series, so they diff against each other + previous = queries.previous_run(session, latest) + assert previous.id == run_id_for(ZE, 100) + + +def test_reimport_skips_then_force_updates(app): + import_runs(_zebrunner([]), _radiator(app), env="ci") + again = import_runs(_zebrunner([]), _radiator(app), env="ci") + assert again.imported == 0 and again.skipped_existing == 2 + + forced = import_runs(_zebrunner([]), _radiator(app), env="ci", force=True) + assert forced.imported == 2 + with app.state.sessionmaker() as session: + assert len(queries.list_runs(session)) == 2 + + +def test_dry_run_writes_nothing(app): + stats = import_runs(_zebrunner([]), None, since=None, fetch_logs=False) + assert stats.imported == 2 + # no env given: the prod-only aragorn agent marks these as prod runs + assert stats.unknown_env == 0 + with app.state.sessionmaker() as session: + assert queries.list_runs(session) == [] + + +def test_filters(app): + from datetime import datetime, timezone + + stats = import_runs( + _zebrunner([]), + _radiator(app), + env="ci", + since=datetime(2025, 3, 2, tzinfo=timezone.utc), + ) + assert stats.launches == 1 and stats.imported == 1 + + +def test_conversion_details(): + assert suite_from_name("sprint_4_tests: 2025_03_02_06_00") == "sprint_4_tests" + assert suite_from_name("odd name") == "odd name" + # an asset from a target-override run: the single agent drives the status + result = convert_test(acceptance_test(5, "Asset_5", {"aragorn": "FAILED"}), None) + assert result.status == "FAILED" + # malformed report values are dropped, not fatal + report = { + "result": {"ars": {"status": "PASSED", "actual_output": {"ars_rank": "n/a"}}} + } + result = convert_test(acceptance_test(6, "Asset_6", {"ars": "PASSED"}), report) + assert result.agents[0].rank is None + + +def test_env_inference(): + prod = convert_test( + acceptance_test(7, "A", {"ars": "PASSED", "arax": "PASSED"}), None + ) + shepherd = convert_test( + acceptance_test(8, "A", {"ars": "PASSED", "shepherd-arax": "PASSED"}), None + ) + assert infer_env([prod]) == "prod" + # dev, ci and test all use the shepherd agents: can't tell which + assert infer_env([shepherd]) is None + + +def test_a_partly_imported_run_is_imported_again(app, api): + """A run that exists but never got its end time stopped partway, and a + re-run finishes it instead of skipping it.""" + run_id = run_id_for(ZE, 101) + api.post( + "/api/ingest/runs", + json={ + "run_id": str(run_id), + "suite": "sprint_4_tests", + "started_at": "2025-03-02T06:00:00Z", + "origin": "zebrunner_import", + }, + ) + stats = import_runs(_zebrunner([]), _radiator(app), env="ci") + assert stats.imported == 2 and stats.skipped_existing == 0 + with app.state.sessionmaker() as session: + run = queries.get_run(session, run_id) + assert run.ended_at is not None and len(run.assets) == 2 + + +def test_zebrunner_errors_say_what_zebrunner_said(): + def handler(request: httpx.Request): + if request.url.path == "/api/iam/v1/auth/refresh": + return httpx.Response(200, json={"authToken": "access"}) + return httpx.Response( + 400, json={"message": "Required parameter 'projectId' is not present"} + ) + + zebrunner = Zebrunner(ZE, "refresh", transport=httpx.MockTransport(handler)) + with pytest.raises(ZebrunnerError) as error: + next(zebrunner.launches()) + message = str(error.value) + assert "/api/reporting/v1/launches" in message and "answered 400" in message + assert "projectId" in message and "pass --project-id" in message + + +def _epoch_ms(text): + return int(datetime.fromisoformat(text.replace("Z", "+00:00")).timestamp() * 1000) + + +def fake_legacy_zebrunner(calls, logs=True): + """An older Zebrunner (eg 1.28): no /v1 read API; launches and tests come + from the legacy search endpoints, in `results`, with the suite nested and + times in epoch millis; and test logs aren't where the import looks.""" + + def handler(request: httpx.Request): + calls.append((request.method, request.url.path, dict(request.url.params))) + path = request.url.path + if path == "/api/iam/v1/auth/refresh": + return httpx.Response(200, json={"authToken": "access"}) + if path == "/api/reporting/v1/launches": + return httpx.Response( + 404, + json={"status": 404, "error": "Not Found", "path": "/v1/launches"}, + ) + if path == "/api/reporting/api/tests/runs/search": + page, size = int(request.url.params["page"]), int( + request.url.params["pageSize"] + ) + runs = [ + { + "id": launch["id"], + "testSuite": {"name": launch["name"]}, + "startedAt": _epoch_ms(launch["startedAt"]), + "elapsed": 3000, + } + for launch in LAUNCHES + ] + return httpx.Response( + 200, json={"results": runs[(page - 1) * size : page * size]} + ) + if path.endswith("/logs") and "/v1/test-runs/" in path and logs: + test_id = int(path.split("/")[7]) + entries = [ + {"level": "INFO", "message": log["message"], "timestamp": 1791954010} + for log in LOGS.get(test_id, []) + ] + # one entry per page, to exercise the paging (no ids to go by) + page = int(request.url.params["page"]) + return httpx.Response( + 200, + json={ + "page": page, + "totalPages": max(len(entries), 1), + "results": entries[page - 1 : page], + }, + ) + if path == "/api/reporting/api/tests/search" and request.method == "POST": + body = json.loads(request.content) + page, size = body["page"], body["pageSize"] + tests = TESTS[body["testRunId"]][(page - 1) * size : page * size] + return httpx.Response(200, json={"results": tests}) + return httpx.Response(404, json={"status": 404, "error": "Not Found"}) + + return httpx.MockTransport(handler) + + +def test_import_from_an_older_zebrunner(app, caplog): + calls = [] + zebrunner = Zebrunner(ZE, "refresh", transport=fake_legacy_zebrunner(calls)) + caplog.set_level("INFO", logger="zebrunner-import") + stats = import_runs(zebrunner, _radiator(app), env="ci", workers=1) + + assert zebrunner.api == "legacy" + assert stats.imported == 2 and not stats.failed + # the reports come from the legacy logs, paged by number + assert stats.with_report == 1 # the one test with a logged report + assert "doesn't serve test logs" not in caplog.text + assert "Zebrunner legacy launch fields: elapsed, id, startedAt, testSuite" in ( + caplog.text + ) + assert any(path == "/api/reporting/api/tests/search" for _, path, _ in calls) + + with app.state.sessionmaker() as session: + latest = queries.get_run(session, run_id_for(ZE, 101)) + assert latest.suite == "sprint_4_tests" + assert latest.started_at == datetime(2025, 3, 2, 6, tzinfo=timezone.utc) + # elapsed (seconds) gives the end + assert (latest.ended_at - latest.started_at).total_seconds() == 3000 + assert {a.asset_id for a in latest.assets} == {"Asset_1", "Asset_2"} + one = next(a for a in latest.assets if a.asset_id == "Asset_1") + ars = next(a for a in one.agents if a.agent == "ars") + assert (ars.rank, ars.pk) == (3, "merged-1") + + +def test_an_older_zebrunner_without_logs_imports_without_reports(app, caplog): + zebrunner = Zebrunner( + ZE, "refresh", transport=fake_legacy_zebrunner([], logs=False) + ) + stats = import_runs(zebrunner, _radiator(app), env="ci", workers=1) + assert stats.imported == 2 and not stats.failed + assert stats.with_report == 0 + assert caplog.text.count("doesn't serve test logs") == 1 + + +def test_the_api_can_be_chosen(): + calls = [] + zebrunner = Zebrunner( + ZE, "refresh", transport=fake_legacy_zebrunner(calls), api="legacy" + ) + assert [launch["id"] for launch in zebrunner.launches()] == [101, 100, 99] + # straight to the legacy API, never trying v1 + assert all("/v1/launches" not in path for _, path, _ in calls) + + +# --- environments from the ARS ------------------------------------------------- + + +def fake_ars(known: dict, down=()): + """ARSs by host: `known` maps a host to the pks it has; hosts in `down` + can't be reached.""" + + def handler(request: httpx.Request): + host = request.url.host + if host in down: + raise httpx.ConnectError("Connection refused", request=request) + pk = request.url.path.rsplit("/", 1)[-1] + if pk in known.get(host, ()): + return httpx.Response(200, json={"message": pk, "status": "Done"}) + return httpx.Response(404, json={"error": "not found"}) + + return httpx.MockTransport(handler) + + +HOSTS = { + "dev": "https://ars.dev.example", + "ci": "https://ars.ci.example", + "test": "https://ars.test.example", +} + + +def test_report_pks_prefer_the_parent(): + reports = { + 1: {"pks": {"ars": "merged-1", "parent_pk": "parent-1"}}, + 2: {"pks": {"parent_pk": "parent-2", "aragorn": "child-2"}}, + 3: {"pks": {}}, + } + assert report_pks(reports) == ["parent-1", "parent-2", "merged-1"] + + +def test_a_runs_environment_comes_from_the_ars_that_has_its_pk(app, caplog): + caplog.set_level("INFO", logger="zebrunner-import") + resolver = ArsEnvResolver( + HOSTS, transport=fake_ars({"ars.ci.example": {"parent-1"}}) + ) + stats = import_runs( + _zebrunner([]), _radiator(app), workers=1, env_resolver=resolver + ) + assert stats.env_from_pk == 1 + assert ( + "launch 101 (sprint_4_tests: 2025_03_02_06_00): pk parent-1 is on the ci ARS" + in (caplog.text) + ) + with app.state.sessionmaker() as session: + assert queries.get_run(session, run_id_for(ZE, 101)).env == "ci" + # no logged report, so no pk: back to the agents (aragorn is prod-only) + assert queries.get_run(session, run_id_for(ZE, 100)).env == "prod" + assert stats.envs == {"ci": 1, "prod": 1} + + +def test_pks_no_ars_has_fall_back(app, caplog): + caplog.set_level("INFO", logger="zebrunner-import") + resolver = ArsEnvResolver(HOSTS, transport=fake_ars({})) + stats = import_runs( + _zebrunner([]), _radiator(app), workers=1, env_resolver=resolver + ) + assert stats.env_from_pk == 0 + assert "none of its pks are on any ARS" in caplog.text + with app.state.sessionmaker() as session: + assert queries.get_run(session, run_id_for(ZE, 101)).env == "prod" + + +def test_an_unreachable_ars_is_skipped_after_one_warning(caplog): + resolver = ArsEnvResolver( + HOSTS, + transport=fake_ars( + {"ars.test.example": {"p1", "p2"}}, down={"ars.dev.example"} + ), + ) + assert resolver.env_for({1: {"pks": {"parent_pk": "p1"}}}) == ("test", "p1") + assert resolver.env_for({1: {"pks": {"parent_pk": "p2"}}}) == ("test", "p2") + assert caplog.text.count("Can't reach the dev ARS") == 1 + + +def test_ars_hosts_come_from_smartapi(): + registry = { + "hits": [ + { + "info": {"x-translator": {"infores": "infores:ars"}}, + "servers": [ + {"url": "https://ars-dev.example/", "x-maturity": "development"}, + {"url": "https://ars.ci.example", "x-maturity": "staging"}, + {"url": "https://ars.test.example", "x-maturity": "testing"}, + {"url": "https://ars-prod.example", "x-maturity": "production"}, + ], + }, + { + "info": {"x-translator": {"infores": "infores:aragorn"}}, + "servers": [ + {"url": "https://aragorn.example", "x-maturity": "staging"} + ], + }, + ] + } + client = httpx.Client( + transport=httpx.MockTransport(lambda r: httpx.Response(200, json=registry)) + ) + assert ars_hosts_from_smartapi(client) == { + "dev": "https://ars-dev.example", + "ci": "https://ars.ci.example", + "test": "https://ars.test.example", + "prod": "https://ars-prod.example", + } diff --git a/radiator/web.py b/radiator/web.py new file mode 100644 index 0000000..367ce93 --- /dev/null +++ b/radiator/web.py @@ -0,0 +1,780 @@ +"""The dashboard's pages. Everything here requires the shared login.""" + +import json +import uuid +from dataclasses import dataclass +from datetime import date, datetime, time, timedelta, timezone +from typing import Optional +from urllib.parse import urlencode + +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import HTMLResponse +from markupsafe import Markup, escape +from sqlalchemy.orm import Session + +from radiator import charts, cycles, queries +from radiator import grid as run_grid +from radiator.api import get_session +from radiator.auth import is_logged_in + + +class LoginRequired(Exception): + """Raised for a page request without a session; redirects to /login.""" + + +def require_login(request: Request) -> None: + if not is_logged_in(request): + raise LoginRequired() + + +router = APIRouter(dependencies=[Depends(require_login)]) + +STATUS_LABELS = { + "PASSED": ("✓", "Passed"), + "FAILED": ("✕", "Failed"), + "NO_RESULTS": ("∅", "No results"), + "ERROR": ("!", "Error"), + "SKIPPED": ("–", "Skipped"), +} + +# Where a pk is opened: the ARAX UI on ci, for every environment, as the +# harness has always linked them. +PK_VIEWER = "https://arax.ci.transltr.io/?r={pk}" + +TREND_WINDOWS = [30, 90, 180, 365] + +# Page sizes for the runs list. No "all": there can be thousands of runs. +RUNS_PER_PAGE = [25, 50, 100] +DEFAULT_RUNS_PER_PAGE = 25 + +# Page sizes for a run's results grid; "all" shows every asset on one page. +RESULTS_PER_PAGE = [50, 100, 250] +DEFAULT_RESULTS_PER_PAGE = 50 + + +@dataclass +class Page: + """One page of a list, and where it sits in the whole.""" + + number: int + pages: int + per_page: Optional[int] # None: everything on one page + total: int + + @property + def start(self) -> int: + """1-based position of the first item shown (0 when there are none).""" + if not self.total: + return 0 + return 1 if self.per_page is None else (self.number - 1) * self.per_page + 1 + + @property + def offset(self) -> int: + """How many items come before this page.""" + return 0 if self.per_page is None else (self.number - 1) * self.per_page + + @property + def end(self) -> int: + if self.per_page is None: + return self.total + return min(self.number * self.per_page, self.total) + + +def make_page( + total: int, + page: int, + per_page: Optional[str], + sizes: list[int], + default: int, + allow_all: bool = False, +) -> Page: + """Work out which page to show. ``per_page`` is one of ``sizes`` (or + "all", when allowed); anything else gets ``default``. A page past the end + shows the last page.""" + size: Optional[int] + if per_page == "all" and allow_all: + size = None + else: + try: + size = int(per_page) + except (TypeError, ValueError): + size = None + if size not in sizes: + size = default + pages = 1 if size is None else max(1, -(-total // size)) + return Page( + number=min(max(page, 1), pages), pages=pages, per_page=size, total=total + ) + + +def paginate(items: list, page: int, per_page: Optional[str]) -> tuple[list, Page]: + """One page of a run's results grid, sliced from all of them.""" + p = make_page( + len(items), page, per_page, RESULTS_PER_PAGE, DEFAULT_RESULTS_PER_PAGE, True + ) + if p.per_page is not None: + items = items[p.offset : p.offset + p.per_page] + return items, p + + +# --- template helpers ---------------------------------------------------------- + + +def status_badge(status: str) -> Markup: + icon, label = STATUS_LABELS.get(status, ("?", status)) + return Markup( + f'{escape(label)}" + ) + + +def status_bar(counts: dict) -> Markup: + total = sum(counts.values()) + if not total: + return Markup('no results') + segments = [] + for status in queries.STATUSES: + n = counts.get(status, 0) + if not n: + continue + label = STATUS_LABELS[status][1] + tip = f"{label}\n{n} of {total} ({n / total:.0%})" + segments.append( + f'' + ) + return Markup( + f'' + ) + + +def _counts_text(counts: dict) -> str: + return ", ".join( + f"{counts[s]} {STATUS_LABELS[s][1].lower()}" + for s in queries.STATUSES + if counts.get(s) + ) + + +def params_text(params: Optional[dict]) -> str: + """Query parameters as a compact one-liner, eg ``timeout=300 · bypass_cache=true``.""" + if not params: + return "" + return " · ".join( + f"{key}={json.dumps(value) if not isinstance(value, str) else value}" + for key, value in params.items() + ) + + +def pct(value: Optional[float], digits: int = 0) -> str: + return "–" if value is None else f"{value * 100:.{digits}f}%" + + +def when(value: Optional[datetime]) -> str: + if value is None: + return "–" + return value.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + + +def ago(value: Optional[datetime]) -> str: + if value is None: + return "" + seconds = (datetime.now(timezone.utc) - value).total_seconds() + for unit, size in (("d", 86400), ("h", 3600), ("m", 60)): + if seconds >= size: + return f"{int(seconds // size)}{unit} ago" + return "just now" + + +def duration(run) -> str: + if run.ended_at is None: + return "–" + minutes = int((run.ended_at - run.started_at).total_seconds() // 60) + return f"{minutes // 60}h {minutes % 60:02d}m" if minutes >= 60 else f"{minutes}m" + + +def elapsed(run) -> str: + """How long a run has been going (or went, if finished).""" + end = run.ended_at or datetime.now(timezone.utc) + minutes = int((end - run.started_at).total_seconds() // 60) + return f"{minutes // 60}h {minutes % 60:02d}m" if minutes >= 60 else f"{minutes}m" + + +def run_badge(run) -> Markup: + """A running or didn't-finish badge for an open run; nothing once done.""" + state = queries.run_state(run) + if state == queries.RUNNING: + return Markup( + '' + "running" + ) + if state == queries.STALLED: + return Markup( + 'didn't finish" + ) + return Markup("") + + +def pk_url(pk: Optional[str]) -> Optional[str]: + return PK_VIEWER.format(pk=pk) if pk else None + + +def cell_tip(asset, agent) -> str: + lines = [ + f"{agent.agent} · {STATUS_LABELS.get(agent.status, ('', agent.status))[1]}" + ] + if agent.found is not None: + lines.append( + "expected answer found" if agent.found else "expected answer not found" + ) + if agent.rank is not None: + lines.append(f"rank {agent.rank}") + if agent.score is not None: + lines.append(f"score {agent.score:.3g}") + if agent.n_results is not None: + lines.append(f"{agent.n_results:,} results") + if agent.response_time_s is not None: + lines.append(f"{agent.response_time_s:.1f}s") + if agent.message: + lines.append(agent.message) + return "\n".join(lines) + + +def query_string(request: Request, **changes) -> str: + params = dict(request.query_params) + for key, value in changes.items(): + if value is None: + params.pop(key, None) + else: + params[key] = value + return "?" + urlencode(params) if params else "?" + + +def register_template_helpers(templates) -> None: + env = templates.env + env.globals.update( + status_badge=status_badge, + status_bar=status_bar, + pct=pct, + params_text=params_text, + when=when, + ago=ago, + duration=duration, + elapsed=elapsed, + run_badge=run_badge, + run_state=queries.run_state, + trapi_minor=queries.trapi_minor, + pk_url=pk_url, + cell_tip=cell_tip, + agent_color=charts.agent_color, + STATUS_LABELS=STATUS_LABELS, + STATUSES=queries.STATUSES, + NO_ENV=queries.NO_ENV, + query_string=query_string, + ) + + +def render(request: Request, template: str, **context) -> HTMLResponse: + return request.app.state.templates.TemplateResponse(request, template, context) + + +# --- pages ------------------------------------------------------------------- + + +@router.get("/", response_class=HTMLResponse) +def runs_page( + request: Request, + suite: Optional[str] = None, + env: Optional[str] = None, + page: int = 1, + per_page: Optional[str] = None, + session: Session = Depends(get_session), +): + total = queries.count_runs(session, suite=suite, env=env) + runs_page = make_page(total, page, per_page, RUNS_PER_PAGE, DEFAULT_RUNS_PER_PAGE) + # only this page's runs come out of the database + runs = queries.list_runs( + session, + suite=suite, + env=env, + limit=runs_page.per_page, + offset=runs_page.offset, + ) + run_ids = [run.id for run in runs] + return render( + request, + "runs.html", + nav="runs", + live_version=queries.live_status(session)["version"], + runs=runs, + counts=queries.status_counts(session, run_ids), + perf_counts=queries.performance_counts(session, run_ids), + pass_rate=queries.pass_rate, + options=queries.filter_options(session), + suite=suite, + env=env, + runs_page=runs_page, + per_page_options=RUNS_PER_PAGE, + total=total, + ) + + +def _load_run(session: Session, run_id: uuid.UUID): + run = queries.get_run(session, run_id) + if run is None: + raise HTTPException(404, "No such run") + return run + + +@router.get("/runs/{run_id}", response_class=HTMLResponse) +def run_page( + request: Request, + run_id: uuid.UUID, + status: Optional[str] = None, + agent: Optional[str] = None, + expected: Optional[str] = None, + q: Optional[str] = None, + page: int = 1, + per_page: Optional[str] = None, + session: Session = Depends(get_session), +): + run = _load_run(session, run_id) + previous = queries.previous_run(session, run) + if previous is not None: + previous = queries.get_run(session, previous.id) + diff = queries.diff_runs(previous, run) + agents = queries.run_agents(run) + + overall = queries.status_counts(session, [run.id])[run.id] + previous_rate = None + if previous is not None: + previous_rate = queries.pass_rate( + queries.status_counts(session, [previous.id])[previous.id] + ) + + assets = sorted(run.assets, key=lambda a: (a.test_case_id, a.name or a.asset_id)) + if status: + if agent: + assets = [ + a + for a in assets + if any(r.agent == agent and r.status == status for r in a.agents) + ] + else: + assets = [a for a in assets if a.status == status] + if expected: + assets = [a for a in assets if a.expected_output == expected] + if q: + needle = q.lower() + assets = [ + a + for a in assets + if needle + in " ".join( + filter( + None, + [a.name, a.asset_id, a.test_case_id, a.input_curie, a.output_curie], + ) + ).lower() + ] + shown, results_page = paginate(assets, page, per_page) + + return render( + request, + "run.html", + nav="runs", + run=run, + previous=previous, + diff=diff, + agents=agents, + agent_counts=queries.agent_counts(run), + agent_expected_counts=queries.agent_expected_counts(run), + overall=overall, + rate=queries.pass_rate(overall), + previous_rate=previous_rate, + assets=shown, + results_page=results_page, + per_page_options=RESULTS_PER_PAGE, + expected_outputs=sorted( + {a.expected_output for a in run.assets if a.expected_output} + ), + filters=dict(status=status, agent=agent, expected=expected, q=q), + filtered=any([status, expected, q]), + pass_rate=queries.pass_rate, + progress=queries.run_progress(session, run), + trapi_change=queries.trapi_change(previous, run), + ) + + +@router.get("/runs/{run_id}/diff", response_class=HTMLResponse) +def diff_page( + request: Request, + run_id: uuid.UUID, + base: Optional[uuid.UUID] = None, + session: Session = Depends(get_session), +): + run = _load_run(session, run_id) + if base is not None: + base_run = _load_run(session, base) + else: + previous = queries.previous_run(session, run) + base_run = queries.get_run(session, previous.id) if previous else None + diff = queries.diff_runs(base_run, run) + running = run.ended_at is None + if running: + # assets not tested yet aren't dropped, just not reached + diff.changes["removed"] = [] + return render( + request, + "diff.html", + nav="runs", + run=run, + base=base_run, + diff=diff, + kinds=queries.DIFF_KINDS, + running=running, + trapi_change=queries.trapi_change(base_run, run), + ) + + +@router.get("/assets/{test_case_id}/{asset_id}", response_class=HTMLResponse) +def asset_page( + request: Request, + test_case_id: str, + asset_id: str, + env: Optional[str] = None, + all: bool = False, + session: Session = Depends(get_session), +): + envs = queries.asset_envs(session, test_case_id, asset_id) + if not env and envs: + # one env's history, not a mix: the one with the most of it + env = envs[0] + history = queries.asset_history(session, test_case_id, asset_id, env=env) + if history.latest is None: + raise HTTPException(404, "No results for this asset") + + def series_for(attr): + return [ + charts.Series( + name=agent, + color=charts.agent_color(agent), + points=[ + ( + run.started_at, + getattr(history.agent_result(asset, agent), attr, None), + ) + for run, asset in history.points + ], + ) + for agent in history.agents + ] + + rank_series = [ + s for s in series_for("rank") if any(y is not None for _, y in s.points) + ] + count_series = [ + s for s in series_for("n_results") if any(y is not None for _, y in s.points) + ] + return render( + request, + "asset.html", + nav="runs", + history=history, + envs=envs, + env=env, + table_runs=list(reversed(history.points))[: None if all else 10], + show_all=all, + rank_chart=( + charts.line_chart( + rank_series, + invert=True, + y_min=1, + y_format=lambda v: f"{v:,.0f}", + width=charts.WIDTH_HALF, + label="Rank of the expected answer over time", + ) + if rank_series + else None + ), + count_chart=( + charts.line_chart( + count_series, + y_min=0, + y_format=lambda v: f"{v:,.0f}", + width=charts.WIDTH_HALF, + label="Number of results over time", + ) + if count_series + else None + ), + ) + + +@router.get("/trends", response_class=HTMLResponse) +def trends_page( + request: Request, + suite: Optional[str] = None, + env: Optional[str] = None, + days: int = 90, + session: Session = Depends(get_session), +): + options = queries.filter_options(session) + if suite is None and options["suites"]: + latest = queries.list_runs(session, limit=1) + suite = latest[0].suite if latest else options["suites"][0] + env = env or (latest[0].env if latest else None) + days = days if days in TREND_WINDOWS else 90 + points = queries.agent_trends(session, suite, env, days) if suite else [] + + agents = sorted( + {agent for p in points for agent in p.counts}, key=queries.agent_sort_key + ) + expected_outputs = [ + e + for e in ["TopAnswer", "Acceptable", "BadButForgivable", "NeverShow"] + if any(e in p.counts.get(a, {}) for p in points for a in agents) + ] + sorted( + {e for p in points for a in agents for e in p.counts.get(a, {})} + - {"TopAnswer", "Acceptable", "BadButForgivable", "NeverShow"} + ) + + def rate(point, agent, expected=None): + by_expected = point.counts.get(agent, {}) + if expected is None: + merged: dict = {} + for counts in by_expected.values(): + for status, n in counts.items(): + merged[status] = merged.get(status, 0) + n + return queries.pass_rate(merged) + return queries.pass_rate(by_expected.get(expected, {})) + + def chart(expected=None, height=240, width=charts.WIDTH_FULL): + return charts.line_chart( + [ + charts.Series( + agent, + charts.agent_color(agent), + [(p.run.started_at, rate(p, agent, expected)) for p in points], + ) + for agent in agents + ], + y_min=0, + y_max=1, + y_format=lambda v: f"{v * 100:.0f}%", + height=height, + width=width, + label=f"Pass rate by agent{f' for {expected}' if expected else ''}", + ) + + return render( + request, + "trends.html", + nav="trends", + options=options, + suite=suite, + env=env, + days=days, + windows=TREND_WINDOWS, + points=points, + agents=agents, + overall_chart=chart() if points else None, + # the same chart drawn for a phone, where the wide one's text would + # shrink to nothing; CSS shows one or the other + overall_chart_narrow=( + chart(height=220, width=charts.WIDTH_THIRD) if points else None + ), + multiples=( + [ + (e, chart(e, height=200, width=charts.WIDTH_HALF)) + for e in expected_outputs + ] + if points + else [] + ), + rate=rate, + ) + + +@router.get("/performance", response_class=HTMLResponse) +def performance_page( + request: Request, days: int = 180, session: Session = Depends(get_session) +): + days = days if days in TREND_WINDOWS else 180 + history = queries.performance_history(session, days) + panels = [] + for key, rows in history.items(): + series = [ + charts.Series( + "max sustainable concurrency", + "var(--agent-1)", + [ + (run.started_at, perf.max_sustainable_concurrency) + for run, perf in rows + ], + ) + ] + panels.append( + ( + key, + rows, + charts.line_chart( + series, + y_min=0, + y_format=lambda v: f"{v:g}", + height=170, + width=charts.WIDTH_THIRD, + label=f"Max sustainable concurrency for {key}", + ), + ) + ) + return render( + request, + "performance.html", + nav="performance", + panels=panels, + days=days, + windows=TREND_WINDOWS, + ) + + +GRID_WINDOWS = [7, 14, 35, 90] + + +@router.get("/grid", response_class=HTMLResponse) +def grid_page( + request: Request, + suite: Optional[str] = None, + days: int = 14, + session: Session = Depends(get_session), +): + options = queries.filter_options(session) + if suite is None: + # the suite of the latest full acceptance run + latest = queries.list_runs(session, limit=20) + suite = next( + (r.suite for r in latest if r.target is None and r.query_type is None), + options["suites"][0] if options["suites"] else None, + ) + days = days if days in GRID_WINDOWS else 14 + grid = None + if suite: + grid = run_grid.build( + session, + suite, + days=days, + width=charts.WIDTH_FULL, + excluded_envs=request.app.state.settings.excluded_envs, + ) + return render( + request, + "grid.html", + nav="grid", + grid_svg=run_grid.to_svg(grid) if grid and not grid.empty else None, + options=options, + suite=suite, + days=days, + windows=GRID_WINDOWS, + excluded=request.app.state.settings.excluded_envs, + trend_labels=run_grid.TREND_LABELS, + ) + + +# --- run cycles and the weekly digest ---------------------------------------- + + +def sparkline( + values: list, pct: bool = True, width: int = 140, height: int = 30 +) -> Markup: + """A small step line of a test's last runs; this run is the dark dot.""" + points = [(i, v) for i, v in enumerate(values) if v is not None] + if not points: + return Markup('–') + lo, hi = min(v for _, v in points), max(v for _, v in points) + floor = 0.10 if pct else max(hi * 0.2, 1.0) + if hi - lo < floor: + mid = (hi + lo) / 2 + lo, hi = mid - floor / 2, mid + floor / 2 + n = max(len(values), 2) + + def px(i): + return 4 + i * (width - 8) / (n - 1) + + def py(v): + return height - 4 - (v - lo) / (hi - lo) * (height - 8) + + path = [] + for k, (i, v) in enumerate(points): + if k == 0: + path.append(f"M{px(i):.1f},{py(v):.1f}") + else: + path.append(f"H{px(i):.1f}V{py(v):.1f}") + dots = "".join( + f'' + for i, v in points[:-1] + ) + i, v = points[-1] + return Markup( + f'' + f'{dots}' + f'' + ) + + +def verdict_chip(verdict: str) -> Markup: + return Markup( + f'{escape(cycles.VERDICT_LABELS[verdict])}" + ) + + +@router.get("/cycles/{cycle_id}", response_class=HTMLResponse) +def cycle_page( + request: Request, cycle_id: uuid.UUID, session: Session = Depends(get_session) +): + cycle = cycles.get_cycle(session, cycle_id) + if cycle is None: + raise HTTPException(404, "No such cycle") + return render( + request, + "cycle.html", + nav="weekly", + cycle=cycle, + sparkline=sparkline, + verdict_chip=verdict_chip, + history_runs=cycles.HISTORY_RUNS, + ) + + +@router.get("/weekly", response_class=HTMLResponse) +def weekly_page( + request: Request, + week_of: Optional[date] = None, + session: Session = Depends(get_session), +): + """The digest for the 7 days ending ``week_of`` (today by default).""" + today = datetime.now(timezone.utc).date() + end_day = min(week_of or today, today) + until = ( + datetime.now(timezone.utc) + if end_day == today + else datetime.combine(end_day, time.max, tzinfo=timezone.utc) + ) + digest = cycles.digest(session, until=until, days=7) + return render( + request, + "weekly.html", + nav="weekly", + live_version=queries.live_status(session)["version"], + digest=digest, + kinds=cycles.KINDS, + kind_labels=cycles.KIND_LABELS, + verdict_chip=verdict_chip, + recent=cycles.recent_cycles(session, limit=12), + previous_week=(end_day - timedelta(days=7)).isoformat(), + next_week=( + (end_day + timedelta(days=7)).isoformat() if end_day < today else None + ), + ) diff --git a/radiator/zebrunner_import.py b/radiator/zebrunner_import.py new file mode 100644 index 0000000..ed4aec4 --- /dev/null +++ b/radiator/zebrunner_import.py @@ -0,0 +1,925 @@ +"""Import past runs from Zebrunner into the Information Radiator. + + python -m radiator.zebrunner_import \\ + --zebrunner-url https://zebrunner.example.org --refresh-token ... \\ + --radiator-url https://radiator.example.org --radiator-token ... \\ + [--project-id 1] [--env ci] [--since 2025-01-01] [--dry-run] + +Reads Zebrunner's own read API (the same service the harness reports to). +Zebrunner changed it between versions, so the import speaks both (``--api``; +by default it tries the newer one and falls back): + +* v1 (reporting-service 1.4x and later): ``GET /api/reporting/v1/launches`` + (needs ``--project-id``), ``GET .../v1/launches/{id}/tests``, and a test's + logs from ``GET .../v1/test-runs/{id}/tests/{testId}/logs-and-screenshots``. +* legacy (eg 1.28): ``GET /api/reporting/api/tests/runs/search``, + ``POST /api/reporting/api/tests/search``, and a test's logs from + ``GET .../v1/test-runs/{id}/tests/{testId}/logs?page=&pageSize=``, as + Zebrunner's own UI calls them. + +and posts each run to the radiator's ingest API, exactly as the harness does. + +What comes across, and from where: + +* From the test labels the harness set: the test case and asset ids, the + expected output, the CURIEs, and each agent's status. +* From the per-test report the harness logged (the JSON ``TestReport``): each + agent's message, whether the expected answer was found, its rank and score, + and the pks. ``--no-logs`` skips fetching these, which is much faster. +* Not available at all: the number of results, HTTP status and response time + (never recorded), and the environment (never sent to Zebrunner; see + ``--env``). + +Imported runs are marked ``origin=zebrunner_import`` and get a run id derived +from the Zebrunner launch id, so re-running the import updates them instead of +duplicating them. Launches already imported in full are skipped unless +``--force``; one whose import stopped partway is imported again. +Performance tests are skipped: they predate HelmsDeep and carry no summary. +""" + +import argparse +import json +import logging +import sys +import uuid +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Iterator, Optional + +import httpx + +import radiator_schema as schema + +logger = logging.getLogger("zebrunner-import") + +# Labels the harness set on every test; any other label key is an agent. +META_LABELS = { + "TestCase", + "TestAsset", + "ExpectedOutput", + "InputCurie", + "OutputCurie", + "SourceInputCurie", + "TargetInputCurie", + "TestRunTime", + "UserSpawnRate", +} +PERFORMANCE_LABELS = {"TestRunTime", "UserSpawnRate"} +STATUSES = {s.value for s in schema.Status} +# Zebrunner's own test statuses, for tests without an agent label to go by. +# It has no NO_RESULTS or ERROR, so the harness's attempts to finish a test +# with those were refused and the test stayed IN_PROGRESS. +ZEBRUNNER_STATUS = { + "PASSED": "PASSED", + "FAILED": "FAILED", + "SKIPPED": "SKIPPED", + "ABORTED": "ERROR", +} +# Agents that only exist in prod; the shepherd-* agents are dev/ci/test. +PROD_AGENTS = { + "aragorn", + "arax", + "biothings-explorer", + "improving-agent", + "unsecret-agent", + "cqs", +} +IMPORT_NAMESPACE = uuid.UUID("6f1c8a52-1d7e-4a8f-9a55-0b7c3f6d2e11") + + +# v1, read from Zebrunner's images (reporting-service 1.54.1 +# LaunchController, test-execution-logs-service 1.6.1 LogController) +LAUNCHES_PATH = "/api/reporting/v1/launches" +TESTS_PATH = "/api/reporting/v1/launches/{launch_id}/tests" +LOGS_PATH = ( + "/api/reporting/v1/test-runs/{launch_id}/tests/{test_id}/logs-and-screenshots" +) +# legacy, as Zebrunner 1.28's UI calls it +LEGACY_LAUNCHES_PATH = "/api/reporting/api/tests/runs/search" +LEGACY_TESTS_PATH = "/api/reporting/api/tests/search" +LEGACY_LOGS_PATH = "/api/reporting/v1/test-runs/{launch_id}/tests/{test_id}/logs" +V1, LEGACY, AUTO = "v1", "legacy", "auto" + + +class ZebrunnerError(RuntimeError): + """Zebrunner answered with an error; the message has its answer.""" + + def __init__(self, message: str, status_code: Optional[int] = None): + super().__init__(message) + self.status_code = status_code + + +def run_id_for(zebrunner_url: str, launch_id) -> uuid.UUID: + """Stable per launch, so re-importing updates rather than duplicates.""" + return uuid.uuid5(IMPORT_NAMESPACE, f"{zebrunner_url.rstrip('/')}/{launch_id}") + + +def parse_time(value) -> Optional[datetime]: + if value in (None, ""): + return None + if isinstance(value, (int, float)): + # epoch millis + return datetime.fromtimestamp(value / 1000, tz=timezone.utc) + text = str(value).replace("Z", "+00:00") + parsed = datetime.fromisoformat(text) + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + + +def suite_from_name(name: str) -> str: + """The harness named runs ``": "``.""" + suite, sep, _ = (name or "").rpartition(":") + return suite.strip() if sep and suite.strip() else (name or "unknown").strip() + + +# --- Zebrunner client -------------------------------------------------------- + + +def _page_items(payload: dict) -> list: + """A page's items: ``items`` in v1, ``results`` in the legacy API.""" + if not isinstance(payload, dict): + return [] + items = payload.get("items") + if items is None: + items = payload.get("results") + return items or [] + + +def legacy_launch(item: dict) -> dict: + """A legacy test run, with the fields the import reads under their v1 + names. The legacy API names and nests some of them differently.""" + suite = item.get("testSuite") or {} + started = item.get("startedAt") or item.get("startTime") or item.get("createdAt") + ended = item.get("endedAt") or item.get("finishTime") + if ended is None and isinstance(started, (int, float)) and item.get("elapsed"): + ended = started + item["elapsed"] * 1000 # elapsed is in seconds + return { + **item, + "name": item.get("name") or suite.get("name") or item.get("testSuiteName"), + "startedAt": started, + "endedAt": ended, + "build": item.get("build") or (item.get("config") or {}).get("build"), + } + + +class Zebrunner: + """The handful of Zebrunner read calls the import needs, in either of its + API generations.""" + + def __init__( + self, + base_url: str, + refresh_token: str, + project_id: Optional[int] = None, + transport: Optional[httpx.BaseTransport] = None, + api: str = AUTO, + ): + self.base_url = base_url.rstrip("/") + self.refresh_token = refresh_token + self.project_id = project_id + self.api = api + self.client = httpx.Client( + base_url=self.base_url, timeout=60, transport=transport + ) + # turned off for the rest of the import by the first logs request + # Zebrunner can't answer, so that's one warning, not one per test + self.logs_available = True + self._shown_fields = set() + self._authenticate() + + def _authenticate(self): + res = self.client.post( + "/api/iam/v1/auth/refresh", json={"refreshToken": self.refresh_token} + ) + res.raise_for_status() + self.client.headers["Authorization"] = f"Bearer {res.json()['authToken']}" + + def _request(self, method: str, path: str, params=None, body=None) -> dict: + params = {k: v for k, v in (params or {}).items() if v is not None} + if self.project_id is not None: + params.setdefault("projectId", self.project_id) + res = self.client.request(method, path, params=params, json=body) + if res.status_code == 401: + # access tokens are short-lived; an import can outlast one + self._authenticate() + res = self.client.request(method, path, params=params, json=body) + if res.is_error: + # say what Zebrunner said, not just the status: a 400 names the + # parameter it wanted + hint = "" + if ( + res.status_code in (400, 404) + and self.project_id is None + and "/v1/launches" in path + ): + hint = " (Zebrunner's v1 API needs the project: pass --project-id)" + raise ZebrunnerError( + f"{method} {res.request.url} answered {res.status_code}: " + f"{res.text[:500]}{hint}", + res.status_code, + ) + return res.json() + + def _get(self, path: str, **params) -> dict: + return self._request("GET", path, params) + + def _show_fields(self, what: str, item: dict): + """Log the fields of the first item of each kind, once: if the + legacy API names something differently, the dry run shows it.""" + if what not in self._shown_fields and isinstance(item, dict): + self._shown_fields.add(what) + logger.info(f"Zebrunner {what} fields: {', '.join(sorted(item))}") + + def _paged(self, fetch, page_size: int) -> Iterator[dict]: + page = 1 + seen = set() + while True: + items = fetch(page, page_size) + fresh = [item for item in items if item.get("id") not in seen] + if not fresh: + return + for item in fresh: + seen.add(item.get("id")) + yield item + if len(items) < page_size: + return + page += 1 + + def launches(self, page_size: int = 50) -> Iterator[dict]: + if self.api in (V1, AUTO): + try: + first = self._get(LAUNCHES_PATH, page=1, pageSize=page_size) + except ZebrunnerError as e: + if self.api == V1 or e.status_code != 404: + raise + logger.info( + "Zebrunner has no /v1/launches (an older Zebrunner, or a " + "newer one without --project-id); using its legacy API" + ) + self.api = LEGACY + else: + self.api = V1 + yield from self._paged( + lambda page, size: ( + _page_items(first) + if page == 1 + else _page_items( + self._get(LAUNCHES_PATH, page=page, pageSize=size) + ) + ), + page_size, + ) + return + for item in self._paged( + lambda page, size: _page_items( + self._get(LEGACY_LAUNCHES_PATH, page=page, pageSize=size) + ), + page_size, + ): + self._show_fields("legacy launch", item) + yield legacy_launch(item) + + def tests(self, launch_id, page_size: int = 500) -> list[dict]: + if self.api == V1: + return _page_items(self._get(TESTS_PATH.format(launch_id=launch_id))) + tests = list( + self._paged( + lambda page, size: _page_items( + self._request( + "POST", + LEGACY_TESTS_PATH, + body={"testRunId": launch_id, "page": page, "pageSize": size}, + ) + ), + page_size, + ) + ) + if tests: + self._show_fields("legacy test", tests[0]) + return tests + + def _legacy_logs(self, launch_id, test_id, page_size: int = 1000) -> list[dict]: + """Paged by number (``page``, ``totalPages``); entries have no id.""" + logs, page = [], 1 + while True: + payload = self._get( + LEGACY_LOGS_PATH.format(launch_id=launch_id, test_id=test_id), + page=page, + pageSize=page_size, + ) + items = _page_items(payload) + logs.extend(items) + pages = payload.get("totalPages") + if not items or (pages is not None and page >= pages): + return logs + if pages is None and len(items) < page_size: + return logs + page += 1 + + def logs(self, launch_id, test_id, page_size: int = 100) -> list[dict]: + """A test's log lines; [] once Zebrunner turns out not to serve them.""" + if not self.logs_available: + return [] + logs, token = [], None + try: + if self.api == LEGACY: + return self._legacy_logs(launch_id, test_id) + while True: + payload = self._get( + LOGS_PATH.format(launch_id=launch_id, test_id=test_id), + maxPageSize=page_size, + pageToken=token, + ) + logs.extend(_page_items(payload)) + token = (payload.get("_meta") or {}).get("nextPageToken") + if not token or not _page_items(payload): + return logs + except ZebrunnerError as e: + if e.status_code not in (404, 405): + raise + self.logs_available = False + logger.warning( + f"This Zebrunner doesn't serve test logs where the import looks " + f"({str(e)[:300]}); importing without ranks, scores and pks " + "(as with --no-logs)." + ) + return [] + + +# --- environments, from the ARS ---------------------------------------------- + +SMARTAPI_URL = "https://smart-api.info/api/query?limit=1000&q=TRAPI" +# SmartAPI's x-maturity for each environment, as the harness maps them +MATURITY_ENV = { + "development": "dev", + "staging": "ci", + "testing": "test", + "production": "prod", +} + + +def ars_hosts_from_smartapi(client: httpx.Client) -> dict[str, str]: + """Each environment's ARS, from the SmartAPI registry (as the harness + finds it).""" + res = client.get(SMARTAPI_URL) + res.raise_for_status() + hosts: dict[str, str] = {} + for hit in res.json().get("hits", []): + infores = (hit.get("info") or {}).get("x-translator", {}).get("infores") + if infores != "infores:ars": + continue + for server in hit.get("servers") or []: + env = MATURITY_ENV.get(server.get("x-maturity")) + if env and server.get("url"): + hosts.setdefault(env, server["url"].rstrip("/")) + return hosts + + +def report_pks(reports: dict, limit: int = 3) -> list[str]: + """A few distinct pks from a launch's reports, parent pks first.""" + parents, others = [], [] + for report in reports.values(): + pks = report.get("pks") or {} + for key, pk in pks.items(): + if not isinstance(pk, str) or not pk: + continue + (parents if key == "parent_pk" else others).append(pk) + seen, out = set(), [] + for pk in parents + others: + if pk not in seen: + seen.add(pk) + out.append(pk) + if len(out) == limit: + break + return out + + +class ArsEnvResolver: + """Which environment a launch ran in, from where its pks live: a pk only + exists on the ARS that made it, and the harness asked the ARS to retain + every response.""" + + def __init__( + self, + hosts: dict[str, str], + transport: Optional[httpx.BaseTransport] = None, + timeout: float = 30, + ): + self.hosts = dict(hosts) + self.client = httpx.Client( + timeout=timeout, transport=transport, follow_redirects=True + ) + self.unreachable: set[str] = set() + self._last: Optional[str] = None + + def _has(self, env: str, pk: str) -> bool: + url = f"{self.hosts[env]}/ars/api/messages/{pk}" + try: + res = self.client.get(url, params={"trace": "y"}) + except httpx.HTTPError as e: + self.unreachable.add(env) + logger.warning( + f"Can't reach the {env} ARS at {self.hosts[env]} ({e}); not " + "looking pks up there for the rest of the import." + ) + return False + if res.status_code != 200: + return False + try: + return isinstance(res.json(), dict) + except ValueError: + return False + + def env_for(self, reports: dict) -> tuple[Optional[str], Optional[str]]: + """(environment, the pk that showed it), or (None, None).""" + for pk in report_pks(reports): + # runs come in order, and neighbours mostly share an environment + order = sorted(self.hosts, key=lambda env: env != self._last) + for env in order: + if env not in self.unreachable and self._has(env, pk): + self._last = env + return env, pk + return None, None + + +# --- conversion ---------------------------------------------------------------- + + +@dataclass +class ImportStats: + launches: int = 0 + imported: int = 0 + skipped_existing: int = 0 + skipped_empty: int = 0 + assets: int = 0 + with_report: int = 0 + performance_tests_skipped: int = 0 + unknown_env: int = 0 + env_from_pk: int = 0 + envs: dict = field(default_factory=dict) + failed: list = field(default_factory=list) + + +def labels_of(test: dict) -> dict[str, str]: + labels = {} + for label in test.get("labels") or []: + if isinstance(label, dict) and label.get("key") is not None: + labels[str(label["key"])] = ( + None if label.get("value") is None else str(label["value"]) + ) + return labels + + +def find_report(logs: list[dict]) -> Optional[dict]: + """The harness's TestReport: the last log line that parses as one.""" + for log in reversed(logs): + try: + data = json.loads(log.get("message") or "") + except (TypeError, ValueError): + continue + if isinstance(data, dict) and isinstance(data.get("result"), dict): + return data + return None + + +def _number(value, kind): + try: + return None if value is None else kind(value) + except (TypeError, ValueError): + return None + + +def expected_answer(agent_report: dict) -> tuple: + """(found, rank, score), as the harness's collector reads them.""" + actual = agent_report.get("actual_output") + if not isinstance(actual, dict) or not actual: + status = agent_report.get("status") + return (False if status == "NO_RESULTS" else None), None, None + rank = actual.get("ars_rank") + if rank is None: + rank = actual.get("ara_rank") + score = actual.get("ars_score") + if score is None: + score = actual.get("ara_score") + found = actual.get("found") + if found is None: + found = rank is not None or score is not None + return bool(found), _number(rank, int), _number(score, float) + + +def convert_test(test: dict, report: Optional[dict]) -> Optional[schema.AssetResult]: + labels = labels_of(test) + if PERFORMANCE_LABELS & labels.keys(): + return None + agents_status = { + key: value + for key, value in labels.items() + if key not in META_LABELS and value in STATUSES + } + report_agents = (report or {}).get("result") or {} + pks = (report or {}).get("pks") or {} + + agents = [] + for agent in sorted(set(agents_status) | set(report_agents)): + detail = report_agents.get(agent) or {} + status = agents_status.get(agent) or detail.get("status") + if status not in STATUSES: + continue + found, rank, score = expected_answer({**detail, "status": status}) + pk = pks.get(agent) + agents.append( + schema.AgentResult( + agent=agent, + status=status, + message=detail.get("message"), + found=found, + rank=rank, + score=score, + pk=None if pk in (None, "", "None") else str(pk), + expected_nodes_found=detail.get("expected_nodes_found"), + ) + ) + + # the overall status the harness gave the test: the ARS's, or the single + # override agent's; Zebrunner's own status only as a fallback + by_agent = {a.agent: a.status for a in agents} + if "ars" in by_agent: + status = by_agent["ars"] + elif len(by_agent) == 1: + status = next(iter(by_agent.values())) + else: + status = ZEBRUNNER_STATUS.get(test.get("status"), "SKIPPED") + + pathfinder = "SourceInputCurie" in labels + parent_pk = pks.get("parent_pk") + return schema.AssetResult( + test_case_id=labels.get("TestCase") or test.get("testClass") or "unknown", + asset_id=labels.get("TestAsset") or f"zebrunner-test-{test.get('id')}", + kind="pathfinder" if pathfinder else "acceptance", + name=test.get("name"), + expected_output=labels.get("ExpectedOutput"), + input_curie=labels.get("SourceInputCurie" if pathfinder else "InputCurie"), + output_curie=labels.get("TargetInputCurie" if pathfinder else "OutputCurie"), + status=status, + parent_pk=None if parent_pk in (None, "", "None") else str(parent_pk), + agents=agents, + details=(report or {}).get("test_details") or {}, + ) + + +def infer_env(results: list[schema.AssetResult]) -> Optional[str]: + agents = {a.agent for r in results for a in r.agents} + if agents & PROD_AGENTS: + return "prod" + return None + + +def convert_launch( + zebrunner_url: str, + launch: dict, + tests: list[dict], + reports: dict, + env: Optional[str], + stats: ImportStats, +) -> Optional[schema.RunPayload]: + results = [] + for test in tests: + result = convert_test(test, reports.get(test.get("id"))) + if result is None: + stats.performance_tests_skipped += 1 + continue + results.append(result) + if not results: + return None + + # the same asset can appear twice if a test was retried; keep the last + deduped = {(r.test_case_id, r.asset_id): r for r in results} + results = list(deduped.values()) + + started_at = parse_time(launch.get("startedAt")) or datetime.now(timezone.utc) + ended_at = parse_time(launch.get("endedAt")) + run_env = env or infer_env(results) + if run_env is None: + stats.unknown_env += 1 + counts: dict = {} + for result in results: + counts[result.status.value] = counts.get(result.status.value, 0) + 1 + + return schema.RunPayload( + run=schema.RunCreate( + run_id=run_id_for(zebrunner_url, launch["id"]), + suite=suite_from_name(launch.get("name")), + env=run_env, + harness_version=launch.get("build"), + started_at=started_at, + origin="zebrunner_import", + origin_ref=str(launch["id"]), + ), + results=results, + finish=( + schema.RunFinish(ended_at=ended_at, counts=counts) if ended_at else None + ), + ) + + +# --- radiator side --------------------------------------------------------------- + + +class Radiator: + def __init__(self, base_url: str, token: str, transport=None): + self.client = httpx.Client( + base_url=base_url.rstrip("/"), + headers={"Authorization": f"Bearer {token}"}, + timeout=60, + transport=transport, + ) + + def has_run(self, run_id: uuid.UUID) -> bool: + """Whether the run was imported in full. + + One that exists but never got its end time stopped partway (eg an + upload failed), so it's imported again; the upserts make that safe. + """ + res = self.client.get(f"/api/runs/{run_id}", params={"results": "false"}) + if res.status_code == 404: + return False + res.raise_for_status() + return res.json().get("ended_at") is not None + + def push(self, payload: schema.RunPayload, batch_size: int = 200) -> None: + prefix = schema.INGEST_PREFIX + run_id = payload.run.run_id + self.client.post( + f"{prefix}/runs", json=payload.run.model_dump(mode="json") + ).raise_for_status() + for i in range(0, len(payload.results), batch_size): + batch = schema.ResultBatch(results=payload.results[i : i + batch_size]) + self.client.post( + f"{prefix}/runs/{run_id}/results", json=batch.model_dump(mode="json") + ).raise_for_status() + if payload.finish is not None: + self.client.patch( + f"{prefix}/runs/{run_id}", json=payload.finish.model_dump(mode="json") + ).raise_for_status() + + +# --- the import ------------------------------------------------------------------ + + +def import_runs( + zebrunner: Zebrunner, + radiator: Optional[Radiator], + *, + env: Optional[str] = None, + since: Optional[datetime] = None, + until: Optional[datetime] = None, + suite: Optional[str] = None, + fetch_logs: bool = True, + force: bool = False, + workers: int = 8, + limit: Optional[int] = None, + env_resolver: Optional[ArsEnvResolver] = None, +) -> ImportStats: + """Import every matching launch. ``radiator=None`` is a dry run.""" + stats = ImportStats() + selected = [] + for launch in zebrunner.launches(): + started = parse_time(launch.get("startedAt")) + if since and started and started < since: + continue + if until and started and started >= until: + continue + if suite and suite_from_name(launch.get("name")) != suite: + continue + if limit is not None and len(selected) >= limit: + break + selected.append(launch) + # Oldest first: the radiator numbers runs as they arrive, so this keeps + # imported runs' numbers in the order they ran. + epoch = datetime.min.replace(tzinfo=timezone.utc) + selected.sort(key=lambda launch: parse_time(launch.get("startedAt")) or epoch) + for launch in selected: + stats.launches += 1 + name = f"launch {launch.get('id')} ({launch.get('name')})" + run_id = run_id_for(zebrunner.base_url, launch["id"]) + try: + if radiator is not None and not force and radiator.has_run(run_id): + stats.skipped_existing += 1 + logger.info(f"{name}: already imported, skipping") + continue + tests = zebrunner.tests(launch["id"]) + reports = {} + if fetch_logs and tests: + with ThreadPoolExecutor(max_workers=workers) as pool: + found = pool.map( + lambda t: ( + t.get("id"), + find_report(zebrunner.logs(launch["id"], t.get("id"))), + ), + tests, + ) + reports = {test_id: report for test_id, report in found if report} + run_env = env + if run_env is None and env_resolver is not None and reports: + run_env, pk = env_resolver.env_for(reports) + if run_env: + stats.env_from_pk += 1 + logger.info(f"{name}: pk {pk} is on the {run_env} ARS") + else: + logger.info(f"{name}: none of its pks are on any ARS") + payload = convert_launch( + zebrunner.base_url, launch, tests, reports, run_env, stats + ) + if payload is None: + stats.skipped_empty += 1 + logger.info(f"{name}: no acceptance results, skipping") + continue + run_env_name = payload.run.env or "unknown" + stats.envs[run_env_name] = stats.envs.get(run_env_name, 0) + 1 + stats.assets += len(payload.results) + stats.with_report += sum(1 for t in tests if t.get("id") in reports) + if radiator is not None: + radiator.push(payload) + stats.imported += 1 + logger.info( + f"{name}: {'would import' if radiator is None else 'imported'} " + f"{len(payload.results)} assets ({payload.run.suite}, env " + f"{payload.run.env or 'unknown'})" + ) + except Exception as e: + stats.failed.append((launch.get("id"), str(e))) + logger.error(f"{name}: failed: {e}") + return stats + + +def make_env_resolver(overrides: list[str]) -> ArsEnvResolver: + """An ArsEnvResolver for every environment: SmartAPI's ARS hosts, with + ``ENV=URL`` overrides on top.""" + hosts: dict[str, str] = {} + try: + with httpx.Client(timeout=30) as client: + hosts = ars_hosts_from_smartapi(client) + except httpx.HTTPError as e: + if not overrides: + raise + logger.warning(f"Couldn't read the SmartAPI registry ({e}); using --ars") + for spec in overrides: + env, sep, url = spec.partition("=") + if not sep or not env.strip() or not url.strip(): + raise ValueError(f"--ars wants ENV=URL, got {spec!r}") + hosts[env.strip()] = url.strip().rstrip("/") + if not hosts: + raise ValueError("no ARS found in the SmartAPI registry; pass --ars") + logger.info( + "Looking runs' environments up on: " + + ", ".join(f"{env} {url}" for env, url in sorted(hosts.items())) + ) + return ArsEnvResolver(hosts) + + +def _date(text: str) -> datetime: + return parse_time(text if "T" in text else f"{text}T00:00:00+00:00") + + +def cli(argv=None) -> int: + parser = argparse.ArgumentParser( + description="Import past runs from Zebrunner into the Information Radiator." + ) + parser.add_argument("--zebrunner-url", required=True) + parser.add_argument( + "--refresh-token", + required=True, + help="Zebrunner API refresh token (ZE_REFRESH_TOKEN)", + ) + parser.add_argument( + "--project-id", + type=int, + help=( + "Zebrunner project id, which newer Zebrunners (the v1 API) need to " + "list launches: the projectId in the API calls its launches page " + "makes" + ), + ) + parser.add_argument( + "--api", + choices=[AUTO, V1, LEGACY], + default=AUTO, + help=( + "Which Zebrunner API to read: v1 (reporting-service 1.4x and " + "later), legacy (eg 1.28), or auto (try v1, fall back to legacy)" + ), + ) + parser.add_argument( + "--radiator-url", help="Where to import to (omit with --dry-run)" + ) + parser.add_argument("--radiator-token", help="The radiator's API token") + parser.add_argument( + "--env", + help=( + "Environment to record every imported run under. Zebrunner never " + "stored it; without this, each run's is looked up from its pks " + "(see --ars), and failing that, runs with prod-only agents are " + "marked prod and the rest have no environment." + ), + ) + parser.add_argument( + "--ars", + action="append", + default=[], + metavar="ENV=URL", + help=( + "An environment's ARS, for looking runs' pks up, eg " + "ci=https://ars.ci.transltr.io. Repeatable. Defaults to the ARS " + "of each environment in the SmartAPI registry." + ), + ) + parser.add_argument( + "--no-env-lookup", + action="store_true", + help="Don't look runs' pks up on the ARSs to find their environment", + ) + parser.add_argument("--suite", help="Only import runs of this suite") + parser.add_argument( + "--since", type=_date, help="Only runs started on/after this date (YYYY-MM-DD)" + ) + parser.add_argument( + "--until", type=_date, help="Only runs started before this date" + ) + parser.add_argument("--limit", type=int, help="Stop after this many runs") + parser.add_argument( + "--no-logs", + action="store_true", + help="Skip per-test logs: no ranks, scores or pks, but much faster", + ) + parser.add_argument( + "--force", action="store_true", help="Re-import runs that were already imported" + ) + parser.add_argument( + "--workers", type=int, default=8, help="Parallel log requests per run" + ) + parser.add_argument( + "--dry-run", action="store_true", help="Read and convert, but write nothing" + ) + args = parser.parse_args(argv) + + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + if not args.dry_run and not (args.radiator_url and args.radiator_token): + parser.error( + "--radiator-url and --radiator-token are required unless --dry-run" + ) + + zebrunner = Zebrunner( + args.zebrunner_url, args.refresh_token, args.project_id, api=args.api + ) + radiator = ( + None if args.dry_run else Radiator(args.radiator_url, args.radiator_token) + ) + env_resolver = None + if args.env is None and not args.no_env_lookup and not args.no_logs: + # the pks come from the logged reports, so no logs, no lookup + try: + env_resolver = make_env_resolver(args.ars) + except (ValueError, httpx.HTTPError) as e: + logger.error(f"Can't look environments up: {e}") + return 1 + try: + stats = import_runs( + zebrunner, + radiator, + env=args.env, + since=args.since, + until=args.until, + suite=args.suite, + fetch_logs=not args.no_logs, + force=args.force, + workers=args.workers, + limit=args.limit, + env_resolver=env_resolver, + ) + except (ZebrunnerError, httpx.HTTPError) as e: + # eg the login or the list of launches: nothing was imported + logger.error(f"Couldn't read from Zebrunner: {e}") + return 1 + logger.info( + f"Done: {stats.imported} of {stats.launches} runs " + f"{'would be ' if args.dry_run else ''}imported, {stats.assets} assets " + f"({stats.with_report} with a logged report). Skipped: " + f"{stats.skipped_existing} already imported, {stats.skipped_empty} " + f"without acceptance results, {stats.performance_tests_skipped} " + f"performance tests. {stats.unknown_env} runs have no environment." + ) + if stats.envs: + envs = ", ".join(f"{env} {n}" for env, n in sorted(stats.envs.items())) + how = ( + f" ({stats.env_from_pk} found from their pks on the ARS)" + if env_resolver is not None + else "" + ) + logger.info(f"Environments: {envs}{how}") + for launch_id, error in stats.failed: + logger.error(f"Failed: launch {launch_id}: {error}") + return 1 if stats.failed else 0 + + +if __name__ == "__main__": + sys.exit(cli()) diff --git a/radiator_schema/__init__.py b/radiator_schema/__init__.py new file mode 100644 index 0000000..365f91d --- /dev/null +++ b/radiator_schema/__init__.py @@ -0,0 +1,180 @@ +"""The contract between the Test Harness and the Information Radiator. + +Both sides import these models: the harness builds them and posts them, the +radiator validates what it receives against them. Changing a model here changes +the API, so a change that is not purely additive needs ``SCHEMA_VERSION`` +bumped and the radiator taught to read both versions. + +Identity is chosen so that every write is an idempotent upsert, which is what +makes it safe to retry an upload or replay a saved payload: + +* a run is keyed by ``run_id``, a UUID the *client* generates, so re-posting a + run updates it instead of creating a second one; +* a result is keyed by ``(run_id, test_case_id, asset_id)``, the ids from the + Tests repo. They are stable across runs, which is also what lets the + radiator line an asset's results up over time. (``hash_test_asset`` is not: + it uses Python's per-process salted ``hash()``.) + +Fields the harness can't know are ``None``, never a stand-in like ``0`` or +``""``: "not recorded" and "zero results" have to stay distinguishable, +especially in runs imported from Zebrunner, which predate most of this. +""" + +from datetime import datetime +from enum import Enum +from typing import Any, Dict, List, Literal, Optional +from uuid import UUID + +from pydantic import BaseModel, Field + +SCHEMA_VERSION = 1 + +# Every write the harness makes goes under this prefix, authenticated with a +# bearer token. The rest of the radiator sits behind the consortium login. +INGEST_PREFIX = "/api/ingest" + + +class Status(str, Enum): + """Outcome of a test, or of a single agent's part in one. + + Mirrors ``test_harness.utils.AgentStatus``; redeclared so this package + stays importable without the harness. + """ + + PASSED = "PASSED" + FAILED = "FAILED" + NO_RESULTS = "NO_RESULTS" + SKIPPED = "SKIPPED" + ERROR = "ERROR" + + +class RunCreate(BaseModel): + """Opens a run. Posted once, before any results.""" + + run_id: UUID + suite: str + env: Optional[str] = None + # Set when the run overrode the services named in the tests, eg to test a + # locally running ARA. Runs with and without an override are different + # series and must not be compared as if they were the same thing. + target: Optional[str] = None + target_url: Optional[str] = None + query_type: Optional[str] = None + # Parameters sent with every query of the run (TRAPI 2.0's + # Query.parameters, eg timeout or bypass_cache), as given. + query_parameters: Optional[Dict[str, Any]] = None + harness_version: Optional[str] = None + # Where the suite came from: the Tests repo archive URL, or a local dir. + tests_source: Optional[str] = None + started_at: datetime + # "harness" for live runs; the Zebrunner importer marks its runs so the UI + # can say why they are missing fields. + origin: Literal["harness", "zebrunner_import"] = "harness" + # The run's id in the system it came from, for imported runs. + origin_ref: Optional[str] = None + # Set when the run is one step of a run cycle (``test-harness-cycle``): + # acceptance, pathfinder and performance run back to back in one + # environment. The cycle's runs share this id, and the radiator reports + # on them together. + cycle_id: Optional[UUID] = None + # How many assets the run will test, so the dashboard can show progress + # while it's running. None when not known (eg imported runs). + expected_assets: Optional[int] = None + # The TRAPI version the run's queries were written in (eg "2.0.0"), so a + # change of version between runs can be told apart from a regression. + trapi_version: Optional[str] = None + + +class AgentResult(BaseModel): + """One agent's part in one test asset.""" + + agent: str + status: Status + message: Optional[str] = None + # HTTP status, or the ARS's status code for an ARA it fanned out to. + http_status: Optional[int] = None + # Whether the expected output came back anywhere in this agent's results, + # and the rank/score it came back with (ARS sugeno for the ARS, the ARA's + # own analysis score for an ARA). + found: Optional[bool] = None + rank: Optional[int] = None + score: Optional[float] = None + n_results: Optional[int] = None + # Only recorded where the harness can measure it honestly (direct queries). + # ARS fan-out is polled every 10s, so timing it from the harness would + # mostly measure the polling. + response_time_s: Optional[float] = None + pk: Optional[str] = None + # Pathfinder only: which expected path nodes were found. + expected_nodes_found: Optional[str] = None + + +class AssetResult(BaseModel): + """The result of one test asset, across every agent it was sent to.""" + + test_case_id: str + asset_id: str + kind: Literal["acceptance", "pathfinder"] + name: Optional[str] = None + expected_output: Optional[str] = None + predicate: Optional[str] = None + # acceptance: input -> output; pathfinder: source -> target + input_curie: Optional[str] = None + output_curie: Optional[str] = None + # The overall status, driven by the ARS (or the override target). + status: Status + parent_pk: Optional[str] = None + agents: List[AgentResult] = Field(default_factory=list) + # Anything type-specific that doesn't merit a column yet, eg Pathfinder's + # expected path nodes. + details: Dict[str, Any] = Field(default_factory=dict) + + +class PerformanceResult(BaseModel): + """One HelmsDeep run against one host.""" + + test_case_id: str + asset_id: str + host: str + helmsdeep_target: Optional[str] = None + profile: Optional[str] = None + status: Status + # Set when the run never produced a summary at all. + error: Optional[str] = None + exit_code: Optional[int] = None + max_sustainable_concurrency: Optional[float] = None + knee_unsupported: Optional[bool] = None + # None when the run had no checkpoints to pass or fail. + checkpoints_passed: Optional[bool] = None + # HelmsDeep's summary.json, verbatim. The fields above are lifted out of it + # for querying; this stays the authoritative copy. + summary: Dict[str, Any] = Field(default_factory=dict) + + +class ResultBatch(BaseModel): + """A batch of results for an open run. Posted as many times as needed.""" + + results: List[AssetResult] = Field(default_factory=list) + performance: List[PerformanceResult] = Field(default_factory=list) + + +class RunFinish(BaseModel): + """Closes a run.""" + + ended_at: datetime + # Overall status counts, as reported in Slack. + counts: Dict[str, int] = Field(default_factory=dict) + + +class RunPayload(BaseModel): + """Everything about a run in one document. + + This is what the harness saves locally (``--local``, or when an upload + fails) and what ``test-harness-radiator push`` replays. + """ + + schema_version: int = SCHEMA_VERSION + run: RunCreate + results: List[AssetResult] = Field(default_factory=list) + performance: List[PerformanceResult] = Field(default_factory=list) + finish: Optional[RunFinish] = None diff --git a/requirements.txt b/requirements.txt index f8dca51..8295600 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,9 @@ httpx==0.27.0 -pydantic==2.7.1 -# reasoner_pydantic==4.1.6 +# translator_tom needs pydantic>=2.12. +pydantic>=2.12.0 setproctitle==1.3.3 slack_sdk==3.27.2 tqdm==4.66.4 translator-testing-model==0.5.0 -reasoner-validator==4.2.5 +# TRAPI object model: builds the TRAPI 2.0 queries the harness sends. +translator_tom>=2.1.0 diff --git a/setup.py b/setup.py index 3fed4ac..bef7dc4 100644 --- a/setup.py +++ b/setup.py @@ -7,14 +7,16 @@ setup( name="sri-test-harness", - version="0.7.3", + version="0.8.0", author="Max Wang", author_email="max@covar.com", url="https://github.com/TranslatorSRI/TestHarness", description="Translator SRI Test Harness", long_description_content_type="text/markdown", long_description=readme, - packages=["test_harness"], + # radiator_schema is the API contract shared with the Information + # Radiator; it ships with the harness so the two can't drift apart. + packages=["test_harness", "radiator_schema"], include_package_data=True, # The editable local test suites ship with the package so `test-harness # load ` works from a plain (non-editable) install too, not just @@ -22,13 +24,19 @@ package_data={"test_harness": ["test_suites/*.json", "test_suites/README.md"]}, zip_safe=False, license="MIT", - python_requires=">=3.9", + python_requires=">=3.10", entry_points={ "console_scripts": [ "test-harness = test_harness.main:cli", # Runs one suite against several services in turn, so a single # scheduled Job can sweep the ARS and each ARA sequentially. "test-harness-sweep = test_harness.sweep:cli", + # Acceptance, then pathfinder, then performance in one + # environment, and the radiator's board for the whole cycle. + "test-harness-cycle = test_harness.cycle:cli", + # Uploads a run the harness saved instead of sending to the + # Information Radiator (--local, or the radiator was unreachable). + "test-harness-radiator = test_harness.radiator_client:cli", ], }, ) diff --git a/test_harness/acceptance_test_runner.py b/test_harness/acceptance_test_runner.py index e889cd3..064969d 100644 --- a/test_harness/acceptance_test_runner.py +++ b/test_harness/acceptance_test_runner.py @@ -2,6 +2,7 @@ from typing import Any, Dict, List, Optional +from test_harness.trapi import binding_ids from test_harness.utils import AgentReport, AgentStatus @@ -49,9 +50,8 @@ def run_acceptance_pass_fail_analysis( results = sort_by_ars_confidence(results) all_ids = [] for res in results: - for res_node, res_value in res["node_bindings"].items(): - for val in res_value: - ids = str(val["id"]) + for res_value in res["node_bindings"].values(): + for ids in binding_ids(res_value): if ids not in all_ids: all_ids.append(ids) if expect_output == "TopAnswer": @@ -70,8 +70,7 @@ def run_acceptance_pass_fail_analysis( n_perc_ids = [] for res in n_perc_res: for res_value in res["node_bindings"].values(): - for val in res_value: - ids = str(val["id"]) + for ids in binding_ids(res_value): if ids not in n_perc_ids: n_perc_ids.append(ids) # Record up front whether the expected answer came back at all. The @@ -89,11 +88,7 @@ def run_acceptance_pass_fail_analysis( for idx, res in enumerate(results): node_bindings = res.get("node_bindings", {}) for k in node_bindings.keys(): - nb = node_bindings[k] - the_id = None - for c in nb: - the_id = c.get("id") - if the_id == out_curie: + if out_curie in binding_ids(node_bindings[k]): ars_score = None ars_rank = None ara_score = None diff --git a/test_harness/cycle.py b/test_harness/cycle.py new file mode 100644 index 0000000..fbd7de0 --- /dev/null +++ b/test_harness/cycle.py @@ -0,0 +1,470 @@ +"""Run a cycle: acceptance, then pathfinder, then performance, in one environment. + +A run cycle is a weekly look at one environment: the acceptance suite (about +50 minutes), the pathfinder suite, and a HelmsDeep performance run against the +ARS (about 70 minutes), one after another so they never load the services at +the same time. Each step is an ordinary ``test-harness`` run, with its own Slack +report as usual; the runs share a cycle id, and when the last one finishes the +cycle posts the Information Radiator's board for the whole cycle: each test +against its previous run in this environment, the regressions and fixes by +agent, and flags for anything odd. + + test-harness-cycle --download \\ + --acceptance sprint_4_tests --pathfinder pathfinder_tests \\ + --performance performance_tests \\ + --performance_target ars=https://ars.ci.transltr.io \\ + --performance_profile mixed \\ + -- --log_level INFO + +Every step runs even when an earlier one fails. Like the sweep, the exit status +answers "did the runs happen?": non-zero only when a step couldn't be carried +out at all, never because a test failed. + +The weekly digest (``test-harness-radiator digest``) is the same board for +every environment; its Slack message is built here too. +""" + +import logging +import os +import subprocess +import sys +import time +from argparse import ArgumentParser, RawDescriptionHelpFormatter +from dataclasses import dataclass +from typing import List, Optional, Sequence +from uuid import uuid4 + +from test_harness.radiator_client import RadiatorClient +from test_harness.reporter import Reporter +from test_harness.slacker import LocalSlacker, Slacker, radiator_slacker +from test_harness.sweep import ( + DEFAULT_OUTPUT_DIR, + HARNESS, + SUBCOMMAND_FLAGS, + TARGETS_ENV, + _banner, + build_command, + parse_targets, + split_passthrough, +) + +FLAG = "⚑" + + +@dataclass +class Step: + name: str # what the step is, for the log and the report + cmd: List[str] + code: Optional[int] = None + minutes: float = 0.0 + + +def _suite_command( + suite: str, + output_dir: str, + harness_args: Sequence[str], + download: bool, + tests_url: Optional[str], + tests_dir: Optional[str], +) -> List[str]: + cmd = [HARNESS, "--output_dir", output_dir, *harness_args] + if download or tests_url: + cmd += ["download", suite] + if tests_url: + cmd += ["--tests_url", tests_url] + else: + cmd += ["load", suite] + if tests_dir: + cmd += ["--tests_dir", tests_dir] + return cmd + + +def plan( + cycle_id: str, + acceptance: Optional[str] = None, + pathfinder: Optional[str] = None, + performance: Optional[str] = None, + performance_targets: Sequence = (), + performance_profile: Optional[str] = None, + output_dir: str = DEFAULT_OUTPUT_DIR, + harness_args: Sequence[str] = (), + download: bool = False, + tests_url: Optional[str] = None, + tests_dir: Optional[str] = None, +) -> List[Step]: + """The cycle's harness invocations, in order.""" + shared = ["--cycle_id", cycle_id, *harness_args] + steps = [] + for kind, suite in (("acceptance", acceptance), ("pathfinder", pathfinder)): + if suite: + steps.append( + Step( + f"{kind} ({suite})", + _suite_command( + suite, + os.path.join(output_dir, kind), + shared, + download, + tests_url, + tests_dir, + ), + ) + ) + if performance: + profile = ( + ["--performance_profile", performance_profile] + if performance_profile + else [] + ) + for target, url in performance_targets: + steps.append( + Step( + f"performance ({target})", + build_command( + target, + url, + performance, + os.path.join(output_dir, "performance"), + [*shared, *profile], + download, + tests_url, + tests_dir, + ), + ) + ) + return steps + + +def run_steps(steps: List[Step]) -> None: + for position, step in enumerate(steps, start=1): + _banner(f"[{position}/{len(steps)}] {step.name}") + print(f"$ {' '.join(step.cmd)}", flush=True) + started = time.time() + try: + step.code = subprocess.run(step.cmd).returncode + except OSError as e: + print(f"Failed to launch the harness for {step.name}: {e}", flush=True) + step.code = 127 + step.minutes = (time.time() - started) / 60 + print( + f"\n{step.name} finished with exit code {step.code} " + f"after {step.minutes:.1f} min", + flush=True, + ) + + +# --- the Slack messages ------------------------------------------------------------ + + +def _row_line(row: dict) -> str: + what = row["label"] + if row.get("host") is None: + what += f" ({row['suite']})" + return ( + f"• *{what}*: {row['value_text']} {row['unit']}, {row['change_text']} · " + f"{row['detail_text']} · *{row['verdict_label']}*" + ) + + +def _changes_lines(row: dict, limit: int = 4) -> List[str]: + lines = [] + for change in row.get("changes", []): + parts = [] + if change["regressions"]: + parts.append(_count_names(change["regressions"], "regression", limit)) + if change["fixed"]: + parts.append(_count_names(change["fixed"], "fixed", limit)) + lines.append(f" {change['agent']}: " + "; ".join(parts)) + return lines + + +def _count_names(assets: list, word: str, limit: int) -> str: + names = ", ".join(a["name"] or a["asset_id"] for a in assets[:limit]) + more = len(assets) - limit + if more > 0: + names += f" and {more} more" + n = len(assets) + label = word if word == "fixed" or n == 1 else f"{word}s" + return f"{n} {label} ({names})" + + +def cycle_message(report: dict, steps: List[Step], url: Optional[str]) -> str: + """The Slack message for a finished cycle, from the radiator's report.""" + env = (report.get("env") or "mixed environments").upper() + total = sum(step.minutes for step in steps) + ran = [step for step in steps if step.code == 0] + lines = [ + f"*{env} run cycle finished* · {len(ran)} of {len(steps)} steps ran · " + f"{int(total // 60)} h {int(total % 60):02d} min" + ] + for step in steps: + if step.code != 0: + lines.append(f"• {step.name}: couldn't run (exit {step.code})") + rows = report.get("rows", []) + lines += [_row_line(row) for row in rows] + changed = [row for row in rows if row.get("changes")] + if changed: + lines.append("") + lines.append("*Regressions and fixes by agent*") + for row in changed: + lines.append(f" {row['label']}:") + lines += _changes_lines(row) + flags = [flag for row in rows for flag in row.get("flags", [])] + if flags: + lines.append("") + lines.append(f"*{FLAG} Flags*") + lines += [f"• {flag['text']}" for flag in flags] + if url: + lines.append(f"\n<{url}|Open the run cycle in the Information Radiator>") + return "\n".join(lines) + + +def digest_message(digest: dict, url: Optional[str]) -> str: + """The Slack message for the weekly digest.""" + since, until = digest["since"][:10], digest["until"][:10] + lines = [f"*Weekly test status* · {since} to {until}"] + if not digest.get("envs"): + lines.append("No finished runs this week.") + for env in digest.get("envs", []): + name = (env.get("env") or "no environment").upper() + flags = sum(len(row["flags"]) for row in env["rows"]) + lines.append(f"*{name}*" + (f" · {FLAG} {flags}" if flags else "")) + for row in env["rows"]: + lines.append( + f" {row['label']}: {row['value_text']}, {row['change_text']} · " + f"{row['verdict_label']}" + ) + flagged = [ + env for env in digest.get("envs", []) if any(r["flags"] for r in env["rows"]) + ] + if flagged: + lines.append("") + lines.append(f"*{FLAG} Flags*") + for env in flagged: + name = (env.get("env") or "no environment").upper() + for row in env["rows"]: + for flag in row["flags"]: + lines.append(f"• {name}: {flag['text']}") + if url: + lines.append(f"\n<{url}|Open the weekly page in the Information Radiator>") + return "\n".join(lines) + + +def _slacker(output_dir: str, logger): + """Where the radiator's board and digest go: its own channel + (RADIATOR_SLACK_*) if it has one; else SLACK_CHANNEL, unless that is + still Zebrunner's (ZE_* set), which gets nothing of the new radiator's; + then None.""" + own = radiator_slacker(logger) + if own is not None: + return own + if Reporter.is_configured(): + logger.info( + "SLACK_CHANNEL is Zebrunner's while ZE_* is set; set " + "RADIATOR_SLACK_WEBHOOK_URL and RADIATOR_SLACK_CHANNEL to post the " + "new Information Radiator's board and digest to a channel of its own." + ) + return None + if Slacker.is_configured(): + return Slacker() + logger.info(f"Slack isn't configured; saving the reports to '{output_dir}'.") + return LocalSlacker(output_dir=output_dir, logger=logger) + + +def post_cycle_report( + cycle_id: str, + steps: List[Step], + radiator: RadiatorClient, + slacker, + logger: logging.Logger, +) -> bool: + """Post the cycle's board to Slack. Returns False if the radiator didn't + have it (then the per-run reports are all there is).""" + report = radiator.cycle(cycle_id) + if report is None or not report.get("rows"): + logger.warning( + "The Information Radiator has no board for this cycle; " + "only the per-run reports were posted." + ) + return False + message = cycle_message(report, steps, radiator.cycle_url(cycle_id)) + slacker.post_notification(messages=[message]) + board = radiator.cycle_board_png(cycle_id) + if board: + env = report.get("env") or "cycle" + slacker.upload_binary_file( + f"{env}_run_cycle.png", + board, + initial_comment=f"{env.upper()} run cycle: each test against its previous run", + title="Run cycle board", + ) + return True + + +def post_digest( + radiator: RadiatorClient, slacker, logger: logging.Logger, days: int = 7 +) -> bool: + digest = radiator.digest(days) + if digest is None: + logger.error("Couldn't get the weekly digest from the Information Radiator.") + return False + slacker.post_notification(messages=[digest_message(digest, radiator.weekly_url)]) + image = radiator.digest_png(days) + if image: + slacker.upload_binary_file( + "weekly_status.png", + image, + initial_comment="Each environment's latest runs this week", + title="Weekly test status", + ) + return True + + +# --- the command ---------------------------------------------------------------------- + + +def cli(argv: Sequence[str] = None) -> int: + if argv is None: + argv = sys.argv[1:] + cycle_argv, harness_args = split_passthrough(argv) + parser = ArgumentParser( + prog="test-harness-cycle", + formatter_class=RawDescriptionHelpFormatter, + description=( + "Run acceptance, then pathfinder, then performance in one " + "environment, one after another, and post the Information " + "Radiator's board for the whole cycle." + ), + epilog=( + "Anything after a `--` separator is passed to every harness " + "invocation, e.g.:\n\n" + " test-harness-cycle --download --acceptance sprint_4_tests \\\n" + " --pathfinder pathfinder_tests --performance performance_tests \\\n" + " --performance_target ars=https://ars.ci.transltr.io \\\n" + " --performance_profile mixed -- --log_level INFO\n" + ), + ) + parser.add_argument("--acceptance", help="The acceptance suite to run first.") + parser.add_argument("--pathfinder", help="The pathfinder suite to run second.") + parser.add_argument("--performance", help="The performance suite to run last.") + parser.add_argument( + "--performance_target", + action="append", + default=[], + metavar="NAME=URL", + help=( + "The service to load in the performance step, e.g. " + "ars=https://ars.ci.transltr.io. Repeat to run several, in order. " + f"Defaults to the {TARGETS_ENV} environment variable." + ), + ) + parser.add_argument( + "--performance_profile", + help="HelmsDeep's query profile for the performance step, e.g. mixed.", + ) + parser.add_argument( + "--download", + action="store_true", + help="Download the suites instead of reading them from local files.", + ) + parser.add_argument("--tests_url", help="URL to download the suites from.") + parser.add_argument("--tests_dir", help="Directory to read local suites from.") + parser.add_argument( + "--output_dir", + default=DEFAULT_OUTPUT_DIR, + help="Base directory for results; each step gets its own subdirectory.", + ) + args = parser.parse_args(cycle_argv) + + misplaced = [ + arg + for arg in harness_args + for flag in SUBCOMMAND_FLAGS + if arg == flag or arg.startswith(f"{flag}=") + ] + if misplaced: + parser.error( + f"{', '.join(misplaced)} belongs before the `--`: it's an argument " + "of the cycle, which passes it to each step's subcommand." + ) + if "--cycle_id" in harness_args: + parser.error("The cycle sets --cycle_id itself.") + if not (args.acceptance or args.pathfinder or args.performance): + parser.error( + "Nothing to run: pass --acceptance, --pathfinder or --performance." + ) + targets = [] + if args.performance: + try: + targets = parse_targets(args.performance_target) + except ValueError as e: + parser.error(str(e)) + if not targets: + parser.error( + "--performance needs a --performance_target (or " + f"{TARGETS_ENV}), e.g. ars=https://ars.ci.transltr.io" + ) + + logging.basicConfig(level=logging.INFO) + logger = logging.getLogger("test-harness-cycle") + cycle_id = str(uuid4()) + steps = plan( + cycle_id, + acceptance=args.acceptance, + pathfinder=args.pathfinder, + performance=args.performance, + performance_targets=targets, + performance_profile=args.performance_profile, + output_dir=args.output_dir, + harness_args=harness_args, + download=args.download, + tests_url=args.tests_url, + tests_dir=args.tests_dir, + ) + print(f"Run cycle {cycle_id}: {len(steps)} steps", flush=True) + run_steps(steps) + + _banner("CYCLE SUMMARY") + for step in steps: + status = "ok" if step.code == 0 else f"FAILED (exit {step.code})" + print(f" {step.name:<40} {status:<20} {step.minutes:>6.1f} min", flush=True) + + # the radiator the steps reported to: from after the `--`, or the env + radiator = RadiatorClient( + base_url=_passthrough_value(harness_args, "--radiator_url"), + token=_passthrough_value(harness_args, "--radiator_token"), + logger=logger, + ) + slacker = _slacker(args.output_dir, logger) if radiator.enabled else None + if slacker is not None: + # the cycle ran; a Slack failure now is worth a warning, not a crash + try: + post_cycle_report(cycle_id, steps, radiator, slacker, logger) + except Exception as e: + logger.warning(f"Couldn't post the cycle board: {type(e).__name__}: {e}") + elif not radiator.enabled: + logger.info("No Information Radiator configured, so no cycle board.") + return 1 if any(step.code != 0 for step in steps) else 0 + + +def _passthrough_value(harness_args: Sequence[str], flag: str) -> Optional[str]: + """The value of ``flag`` among the harness arguments (``--flag value`` or + ``--flag=value``), or None.""" + for i, arg in enumerate(harness_args): + if arg == flag and i + 1 < len(harness_args): + return harness_args[i + 1] + if arg.startswith(f"{flag}="): + return arg.split("=", 1)[1] + return None + + +def digest_cli(radiator: RadiatorClient, days: int, output_dir: str, logger) -> int: + """``test-harness-radiator digest``: post the weekly digest to Slack.""" + slacker = _slacker(output_dir, logger) + if slacker is None: + return 0 + return 0 if post_digest(radiator, slacker, logger, days) else 1 + + +if __name__ == "__main__": + sys.exit(cli()) diff --git a/test_harness/download.py b/test_harness/download.py index 72918f0..1c0e8d0 100644 --- a/test_harness/download.py +++ b/test_harness/download.py @@ -161,5 +161,3 @@ def download_tests( "https://github.com/NCATSTranslator/Tests/archive/refs/heads/performance_tests.zip", logging.Logger("tester"), ) - for test_case_id, test in tests.items(): - print(type(test)) diff --git a/test_harness/main.py b/test_harness/main.py index 9266b7b..49cbb9d 100644 --- a/test_harness/main.py +++ b/test_harness/main.py @@ -8,19 +8,33 @@ import os import sys import time -from argparse import ArgumentParser +from argparse import ArgumentParser, ArgumentTypeError +from datetime import datetime +from importlib.metadata import PackageNotFoundError, version from urllib.parse import urlparse -from uuid import uuid4 +from uuid import UUID, uuid4 +import httpx from setproctitle import setproctitle +from radiator_schema import RunCreate from test_harness.download import download_tests, load_tests from test_harness.logger import get_logger, setup_logger from test_harness.performance_test_runner import PROFILES +from test_harness.radiator_client import RadiatorClient from test_harness.reporter import LocalReporter, Reporter from test_harness.result_collector import ResultCollector from test_harness.run import run_tests -from test_harness.slacker import LocalSlacker, Slacker +from test_harness.slacker import LocalSlacker, Slacker, radiator_slacker +from test_harness.runner.query_runner import env_map +from test_harness.runner.smart_api_registry import registered_trapi_versions +from test_harness.trapi import ( + AUTO_TRAPI_VERSION, + DEFAULT_TRAPI_VERSION, + parse_query_parameters, + trapi_minor_version, + validate_query_parameters, +) from test_harness.utils import QUERY_TYPE_PREDICATES, filter_tests_by_query_type setproctitle("TestHarness") @@ -45,6 +59,194 @@ def url_type(arg): raise TypeError("Invalid URL") +def uuid_type(arg): + try: + return UUID(arg) + except ValueError: + raise ArgumentTypeError(f"Not a UUID: {arg!r}") + + +def harness_version(): + try: + return version("sri-test-harness") + except PackageNotFoundError: + return None + + +def log_radiator_outcome(radiator, tests, logger): + """What reached the radiator against what the suite had, naming any assets + that never got a result, so a gap in the dashboard can be traced.""" + expected = [ + (test.id, asset.id) for test in tests.values() for asset in test.test_assets + ] + for i, line in enumerate(radiator.outcome(expected)): + (logger.warning if i else logger.info)(line) + + +def save_radiator_run(radiator, output_dir, prefix, logger, force=False): + """Save the run for `test-harness-radiator push` if it wasn't uploaded.""" + if force or not radiator.enabled or radiator.upload_failed: + path = radiator.save(output_dir, prefix=prefix) + logger.info( + f"Saved the run for the Information Radiator to {path}; upload it " + "with `test-harness-radiator push`." + ) + + +def radiator_headline(radiator, collector) -> str: + """Lines for Slack: this run against the previous one in its series (the + acceptance pass rate, each service's top concurrency), linking to what + changed. Empty when the radiator doesn't have the run.""" + if not (collector.has_acceptance_results or collector.has_performance_results): + return "" + summary = radiator.summary() + if not summary: + return "" + lines = [] + if collector.has_acceptance_results and summary.get("pass_rate") is not None: + line = f"Pass rate {summary['pass_rate']:.0%}" + if summary.get("previous_pass_rate") is not None: + delta = (summary["pass_rate"] - summary["previous_pass_rate"]) * 100 + line += f" ({delta:+.1f} pts vs the previous run)" + line += ( + f" · {summary['regressions']} regressions · {summary['fixed']} fixed" + f" · <{radiator.diff_url}|What changed>" + ) + lines.append(line) + if collector.has_performance_results: + lines.extend(_performance_line(p) for p in summary.get("performance") or []) + change = _trapi_change( + summary.get("previous_trapi_version"), summary.get("trapi_version") + ) + if lines and change: + lines.append( + f"TRAPI {change[0]} → {change[1]} since the previous run: some of " + "these changes may come from the version change" + ) + return "".join(f"\n> {line}" for line in lines) + ("\n" if lines else "") + + +def _trapi_change(before, after): + """(from, to) major.minor TRAPI versions when they differ, else None.""" + if not before or not after: + return None + before, after = ".".join(before.split(".")[:2]), ".".join(after.split(".")[:2]) + return (before, after) if before != after else None + + +def _performance_line(perf: dict) -> str: + """One service's top concurrency against its previous run.""" + name = perf.get("helmsdeep_target") or perf["host"] + if perf.get("profile"): + name += f" ({perf['profile']})" + if perf.get("error"): + return f"{name}: run failed: {perf['error']}" + current = perf.get("max_sustainable_concurrency") + if current is None: + line = f"{name}: no stage met the SLO" + else: + line = f"{name}: max sustainable concurrency {current:.1f}" + previous = perf.get("previous_max_sustainable_concurrency") + if previous is not None: + line += f" ({current - previous:+.1f} vs the previous run)" + if perf.get("knee_unsupported"): + line += " (unsupported, see stage warnings)" + if perf.get("checkpoints_passed") is not None: + line += ( + " · checkpoints passed" + if perf["checkpoints_passed"] + else " · checkpoints missed" + ) + return line + + +def post_report( + slacker, + suite, + duration, + queried_envs, + links, + result_summary, + radiator, + collector, + reporter, + prefix, + logger, +): + """A finished run's report in one channel: the summary, the radiator's + charts (with ``radiator``), and the result files.""" + slacker.post_notification( + messages=[ + f"Test Suite: {suite}\nDuration: {duration} | Environment(s): " + f"{','.join(list(queried_envs))}{links}\n{result_summary}" + ] + ) + if radiator is not None: + post_history_charts(radiator, slacker, collector, suite, prefix, logger) + if collector.has_acceptance_results: + slacker.upload_test_results_file( + f"{prefix}{reporter.test_name}", + "json", + collector.acceptance_stats, + ) + slacker.upload_test_results_file( + f"{prefix}{reporter.test_name}", + "csv", + collector.acceptance_csv, + ) + if collector.has_performance_results: + # HelmsDeep's own summary.json is the authoritative result, so it is + # uploaded verbatim alongside its HTML report rather than being + # reformatted into a second, divergent JSON. Each carries the run's + # checkpoint pass/fail as its comment. + for filename, content, comment in collector.render_performance_artifacts(): + filename = f"{prefix}{filename}" + try: + slacker.upload_binary_file(filename, content, initial_comment=comment) + except Exception as e: + logger.warning(f"Failed to upload perf artifact {filename}: {e}") + + +def post_history_charts(radiator, slacker, collector, suite, prefix, logger): + """Post the run's history charts, from the radiator, under the report: + the acceptance pass rate and each service's top concurrency. Best effort: + the report stands without them.""" + charts = [] + if collector.has_acceptance_results: + charts.append( + ( + radiator.history_png, + "pass_rate_history.png", + "Pass-rate history", + f"Pass-rate history for {suite} up to this run", + ) + ) + if collector.has_performance_results: + charts.append( + ( + radiator.performance_png, + "concurrency_history.png", + "Max sustainable concurrency history", + "Max sustainable concurrency history up to this run", + ) + ) + for fetch, filename, title, comment in charts: + png = fetch() + if not png: + continue + try: + slacker.upload_binary_file( + f"{prefix}{filename}", + png, + initial_comment=( + f"{comment} · <{radiator.run_url}|Open in the Information Radiator>" + ), + title=title, + ) + except Exception as e: + logger.warning(f"Failed to post the {title.lower()} chart: {e}") + + def main(args): """Main Test Harness entrypoint.""" qid = str(uuid4())[:8] @@ -76,6 +278,25 @@ def main(args): logger.error(f"No {query_type} tests to run. Exiting.") return EXIT_COULD_NOT_RUN + # The TRAPI version: given, or (auto) what this environment's services are + # registered with, while environments are on different versions. + test_env = next(iter(tests.values())).test_env + args["trapi_version"] = resolve_trapi_version(args, test_env, logger) + # Check the query parameters against the TRAPI version now, so a typo + # stops the run before any query goes out rather than failing every one. + try: + args["query_parameters"] = validate_query_parameters( + args.get("query_parameters"), args["trapi_version"] + ) + except ValueError as e: + logger.error(str(e)) + return EXIT_COULD_NOT_RUN + if args["query_parameters"]: + logger.info( + f"Sending these parameters with every query: " + f"{json.dumps(args['query_parameters'])}" + ) + output_dir = args.get("output_dir") or "test_results" # prefix saved/uploaded result filenames with the override target and the @@ -106,9 +327,18 @@ def main(args): refresh_token=args.get("reporter_access_token"), logger=logger, ) - reporter.get_auth() - test_env = next(iter(tests.values())).test_env - reporter.create_test_run(test_env, args["suite"]) + try: + reporter.get_auth() + reporter.create_test_run(test_env, args["suite"]) + except Exception as e: + # Zebrunner being down mustn't stop the run: the tests still run and + # still reach Slack and the new Information Radiator. + logger.error( + f"Couldn't open a run in the Zebrunner Information Radiator ({e}); " + "continuing without it." + ) + reporter = LocalReporter(logger=logger) + reporter.create_test_run(test_env, args["suite"]) use_local_slacker = local or not Slacker.is_configured() if use_local_slacker: @@ -116,51 +346,152 @@ def main(args): slacker = LocalSlacker(output_dir=output_dir, logger=logger) else: slacker = Slacker() - collector = ResultCollector(test_env, logger, target=args.get("target")) - queried_envs = set() - for test in tests.values(): - queried_envs.add(test.test_env) - slacker.post_notification( - messages=[ - f"Running {args['suite']} ({sum([len(test.test_assets) for test in tests.values()])} tests, {len(tests.values())} queries)...\n<{reporter.base_path}/test-runs/{reporter.test_run_id}|View in the Information Radiator>" - ] + # With RADIATOR_SLACK_* set, the new radiator's reports go to their own + # channel, and SLACK_CHANNEL keeps getting the report it always did. + new_slacker = None if local else radiator_slacker(logger) + # The new Information Radiator runs alongside Zebrunner until it takes + # over. It is additive: without RADIATOR_URL/RADIATOR_TOKEN, or with + # --local, it only records, and the run is saved to --output_dir instead. + radiator = RadiatorClient( + base_url=args.get("radiator_url"), + token=args.get("radiator_token"), + enabled=not local, + logger=logger, + ) + radiator.start_run( + RunCreate( + run_id=uuid4(), + suite=args["suite"], + env=test_env, + target=args.get("target"), + target_url=args.get("target_url"), + query_type=query_type, + query_parameters=args.get("query_parameters"), + cycle_id=args.get("cycle_id"), + trapi_version=args["trapi_version"], + expected_assets=sum(len(test.test_assets) for test in tests.values()), + harness_version=harness_version(), + tests_source=args.get("tests_url") or args.get("tests_dir"), + started_at=datetime.now().astimezone(), + ) + ) + if radiator.enabled and not radiator.upload_failed: + logger.info( + f"Reporting to the Information Radiator at {radiator.base_url}: " + f"{radiator.run_url}" + ) + elif not radiator.enabled: + logger.info( + "Not reporting to the Information Radiator (" + + ("--local" if local else "RADIATOR_URL and RADIATOR_TOKEN aren't set") + + f"); the run will be saved to {output_dir} for " + "`test-harness-radiator push`." + ) + params_note = ( + f"\nQuery parameters: `{json.dumps(args['query_parameters'])}`" + if args.get("query_parameters") + else "" + ) + # Only link to Zebrunner when the run went there: the local stand-in's + # "URL" leads nowhere. + zebrunner_link = ( + "" + if reporter.is_local + else f"\n<{reporter.base_path}/test-runs/{reporter.test_run_id}|View in the Information Radiator>" + ) + radiator_link = ( + f"\n<{radiator.run_url}|View in the new Information Radiator>" + if radiator.run_url + else "" ) - start_time = time.time() - run_tests(tests, reporter, collector, logger, args) - slacker.post_notification( - messages=[ - """Test Suite: {test_suite}\nDuration: {duration} | Environment(s): {envs}\n<{ir_url}|View in the Information Radiator>\n{result_summary}""".format( - test_suite=args["suite"], - duration=round(time.time() - start_time, 2), - envs=(",").join(list(queried_envs)), - ir_url=f"{reporter.base_path}/test-runs/{reporter.test_run_id}", - result_summary=collector.dump_result_summary(), + # Where the reports go: (slacker, links, with the radiator's headline and + # charts, best effort). While Zebrunner is configured, SLACK_CHANNEL is its + # channel and keeps the report it always got; the new radiator's report + # goes only to a channel of its own (RADIATOR_SLACK_*). Without Zebrunner, + # SLACK_CHANNEL is the radiator's. Configured, not reachable: a Zebrunner + # outage mustn't switch the old channel over to the new dashboard. + zebrunner_configured = not use_local_reporter + channels = [] + if zebrunner_configured or new_slacker is None: + channels.append( + ( + slacker, + zebrunner_link if zebrunner_configured else radiator_link, + not zebrunner_configured, + False, ) - ] + ) + if new_slacker is not None: + channels.append((new_slacker, radiator_link, True, True)) + if zebrunner_configured and new_slacker is None and radiator.enabled: + logger.info( + "Reporting to Zebrunner, so SLACK_CHANNEL gets its report only; set " + "RADIATOR_SLACK_WEBHOOK_URL and RADIATOR_SLACK_CHANNEL to post the " + "new Information Radiator's report to a channel of its own." + ) + + def post(action, best_effort): + if not best_effort: + return action() + # the new channel never costs the old one its report + try: + action() + except Exception as e: + logger.warning(f"Couldn't post to the Information Radiator's channel: {e}") + + collector = ResultCollector( + test_env, logger, target=args.get("target"), radiator=radiator ) - if collector.has_acceptance_results: - slacker.upload_test_results_file( - f"{target_prefix}{reporter.test_name}", - "json", - collector.acceptance_stats, + queried_envs = set() + for test in tests.values(): + queried_envs.add(test.test_env) + for channel, links, _, best_effort in channels: + post( + lambda: channel.post_notification( + messages=[ + f"Running {args['suite']} ({sum([len(test.test_assets) for test in tests.values()])} tests, {len(tests.values())} queries, TRAPI {args['trapi_version']})...{params_note}{links}" + ] + ), + best_effort, ) - slacker.upload_test_results_file( - f"{target_prefix}{reporter.test_name}", - "csv", - collector.acceptance_csv, + start_time = time.time() + try: + run_tests(tests, reporter, collector, logger, args) + except BaseException: + # Keep what was collected, but leave the run open: a run that died + # halfway must not read as finished. + radiator.flush() + logger.error("The run crashed; the Information Radiator has it as unfinished.") + log_radiator_outcome(radiator, tests, logger) + save_radiator_run(radiator, output_dir, target_prefix, logger, force=True) + raise + # Close out the radiator before Zebrunner and Slack, whose failures below + # would otherwise cost it the run. + radiator.finish_run(counts=collector.acceptance_report) + log_radiator_outcome(radiator, tests, logger) + save_radiator_run(radiator, output_dir, target_prefix, logger) + + duration = round(time.time() - start_time, 2) + summary = collector.dump_result_summary() + headline = radiator_headline(radiator, collector) + for channel, links, with_radiator, best_effort in channels: + post( + lambda: post_report( + channel, + args["suite"], + duration, + queried_envs, + links, + (headline if with_radiator else "") + summary, + radiator if with_radiator else None, + collector, + reporter, + target_prefix, + logger, + ), + best_effort, ) - if collector.has_performance_results: - # HelmsDeep's own summary.json is the authoritative result, so it is - # uploaded verbatim alongside its HTML report rather than being - # reformatted into a second, divergent JSON. Each carries the run's - # checkpoint pass/fail as its comment. - for filename, content, comment in collector.render_performance_artifacts(): - filename = f"{target_prefix}{filename}" - try: - slacker.upload_binary_file(filename, content, initial_comment=comment) - except Exception as e: - logger.warning(f"Failed to upload perf artifact {filename}: {e}") logger.info("Finishing up test run...") reporter.finish_test_run() @@ -176,6 +507,66 @@ def main(args): return EXIT_OK +def _query_parameters(value: str): + """Parse --query_parameters / QUERY_PARAMETERS: JSON, or @file.""" + try: + return parse_query_parameters(value) + except (ValueError, OSError) as e: + raise ArgumentTypeError(str(e)) + + +def resolve_trapi_version(args, test_env, logger) -> str: + """The TRAPI version to run: as given, or for ``auto``, the newest one the + environment's ARS (or the --target service) is registered with in + SmartAPI that the harness speaks. Falls back to the default, saying why.""" + requested = args.get("trapi_version") or DEFAULT_TRAPI_VERSION + if requested != AUTO_TRAPI_VERSION: + return requested + target = args.get("target") + infores = f"infores:{target.split('infores:')[-1]}" if target else "infores:ars" + maturity = env_map.get(test_env) + why = ( + f"{infores} isn't registered in SmartAPI for {test_env} ({maturity}) " + "with a TRAPI version the harness speaks" + ) + try: + versions = [] + for registered in registered_trapi_versions(infores, maturity): + try: + trapi_minor_version(registered) + except ValueError: + continue + versions.append(registered) + except (httpx.HTTPError, ValueError) as e: + # ValueError: an answer that isn't JSON (eg a proxy's error page) + versions, why = [], f"couldn't read the SmartAPI registry ({e})" + if versions: + others = ( + f" (also registered: {', '.join(versions[1:])})" if versions[1:] else "" + ) + logger.info( + f"TRAPI {versions[0]}: what {infores} is registered with for " + f"{test_env} in SmartAPI{others}" + ) + return versions[0] + logger.warning( + f"--trapi_version auto: {why}; using {DEFAULT_TRAPI_VERSION}. Pass " + "--trapi_version to choose one." + ) + return DEFAULT_TRAPI_VERSION + + +def _trapi_version(value: str) -> str: + """Accept a TRAPI version only if the harness can write queries in it.""" + if value == AUTO_TRAPI_VERSION: + return value + try: + trapi_minor_version(value) + except ValueError as e: + raise ArgumentTypeError(str(e)) + return value + + def cli(): """Parse args and run tests.""" parser = ArgumentParser(description="Translator SRI Automated Test Harness") @@ -247,6 +638,18 @@ def cli(): help="Access token for authentication with the Testing Dashboard", ) + parser.add_argument( + "--radiator_url", + type=url_type, + help="URL of the new Information Radiator (defaults to $RADIATOR_URL)", + ) + + parser.add_argument( + "--radiator_token", + type=str, + help="Ingest token for the new Information Radiator (defaults to $RADIATOR_TOKEN)", + ) + parser.add_argument( "--save_to_dashboard", action="store_true", @@ -287,6 +690,16 @@ def cli(): ), ) + parser.add_argument( + "--cycle_id", + type=uuid_type, + help=( + "Mark the run as one step of a run cycle, whose runs share this " + "id (a UUID). test-harness-cycle sets it; there's no need to pass " + "it by hand." + ), + ) + parser.add_argument( "--performance_profile", type=str.lower, @@ -304,9 +717,29 @@ def cli(): parser.add_argument( "--trapi_version", - type=str, - default="1.6.0", - help="TRAPI (SemVer) version assumed for testing (1.5.0, if not given)", + type=_trapi_version, + default=AUTO_TRAPI_VERSION, + help=( + "TRAPI (SemVer) version to test: queries are written in it and " + "only services registered for it are queried. 2.0.x or 1.6.x, or " + "auto (the default): the version the environment's ARS (or the " + "--target service) is registered with in SmartAPI, falling back " + f"to {DEFAULT_TRAPI_VERSION}." + ), + ) + + parser.add_argument( + "--query_parameters", + type=_query_parameters, + default=os.getenv("QUERY_PARAMETERS"), + help=( + "Parameters sent with every query, as a JSON object or @file, eg " + '\'{"timeout": 300, "bypass_cache": true}\'. TRAPI 2.0 queries carry ' + "them in their parameters object (timeout, log_level, bypass_cache, " + "and any a service defines); TRAPI 1.6 queries take only log_level " + "and bypass_cache. Defaults to $QUERY_PARAMETERS. Not used by " + "performance tests, which send HelmsDeep's own queries." + ), ) parser.add_argument( diff --git a/test_harness/pathfinder_test_runner.py b/test_harness/pathfinder_test_runner.py index 8c6d921..4cb3212 100644 --- a/test_harness/pathfinder_test_runner.py +++ b/test_harness/pathfinder_test_runner.py @@ -1,5 +1,6 @@ from typing import Any, Dict, List +from test_harness.trapi import binding_ids from test_harness.utils import AgentStatus, PathfinderReport @@ -13,9 +14,8 @@ def pathfinder_pass_fail_analysis( found_path_nodes = set() unmatched_paths = set() for analysis in message["results"][0]["analyses"]: - for path_bindings in analysis["path_bindings"].values(): - for path_binding in path_bindings: - path_id = path_binding["id"] + for path_binding in analysis["path_bindings"].values(): + for path_id in binding_ids(path_binding): matching_path_nodes = set() for edge_id in message["auxiliary_graphs"][path_id]["edges"]: edge = message["knowledge_graph"]["edges"][edge_id] diff --git a/test_harness/radiator_client.py b/test_harness/radiator_client.py new file mode 100644 index 0000000..4f50d95 --- /dev/null +++ b/test_harness/radiator_client.py @@ -0,0 +1,390 @@ +"""Client for the Information Radiator's ingest API. + +Runs alongside the Zebrunner ``Reporter`` while the new radiator is brought up. +Unlike the Reporter, it never makes a call per test: results are buffered and +posted in batches as structured records (see ``radiator_schema``). + +An unreachable radiator must never cost a run its results, so nothing here +raises. Every record is kept in ``payload`` whether or not it was uploaded; +when an upload fails the harness saves that payload to disk, and +``test-harness-radiator push`` replays it later. Every write is an upsert on +ids the client chose, so replaying a run that partly made it is safe. +""" + +import json +import logging +import os +import sys +from argparse import ArgumentParser +from datetime import datetime +from typing import List, Optional + +import httpx + +from radiator_schema import ( + INGEST_PREFIX, + AssetResult, + PerformanceResult, + ResultBatch, + RunCreate, + RunFinish, + RunPayload, +) + + +class RadiatorClient: + """Posts a run's results to the Information Radiator.""" + + def __init__( + self, + base_url: Optional[str] = None, + token: Optional[str] = None, + enabled: bool = True, + public_url: Optional[str] = None, + batch_size: int = 50, + logger: logging.Logger = logging.getLogger(__name__), + ): + """Initialize the client. + + With ``enabled=False``, or without a URL and token, nothing is sent: + the client only records, which is what ``--local`` runs want. + + ``public_url`` (or RADIATOR_PUBLIC_URL) is where people open the + radiator, for the links the harness logs and posts to Slack, when it + differs from where the harness uploads to: eg ``http://radiator:8000`` + inside docker compose, but ``http://localhost:8000`` in a browser. + """ + self.base_url = (base_url or os.getenv("RADIATOR_URL") or "").rstrip("/") + self.public_url = ( + public_url or os.getenv("RADIATOR_PUBLIC_URL") or self.base_url + ).rstrip("/") + self.token = token or os.getenv("RADIATOR_TOKEN") + self.enabled = enabled and bool(self.base_url and self.token) + self.batch_size = batch_size + self.logger = logger + self.payload: Optional[RunPayload] = None + # Set by the first failed upload. Nothing is sent after that: the run + # is replayed from the saved payload instead, which avoids a half-run + # in the radiator that looks complete. + self.upload_failed = False + self._pending = ResultBatch() + self._client: Optional[httpx.Client] = None + # what actually reached the radiator, for the end-of-run log + self.sent_results = 0 + self.sent_performance = 0 + self.sent_batches = 0 + + @staticmethod + def is_configured(base_url=None, token=None) -> bool: + """Return True if enough config exists to talk to the radiator.""" + return bool( + (base_url or os.getenv("RADIATOR_URL")) + and (token or os.getenv("RADIATOR_TOKEN")) + ) + + @property + def run_url(self) -> Optional[str]: + """Link to the run in the radiator UI, once a run is open.""" + if not self.enabled or self.payload is None: + return None + return f"{self.public_url}/runs/{self.payload.run.run_id}" + + def start_run(self, run: RunCreate): + """Open a run.""" + self.payload = RunPayload(run=run) + self._send("POST", "/runs", run.model_dump(mode="json"), what="opening the run") + + def add_result(self, result: AssetResult): + """Record one asset's result, uploading once a batch has built up.""" + self.payload.results.append(result) + self._pending.results.append(result) + if len(self._pending.results) >= self.batch_size: + self.flush() + + def add_performance(self, result: PerformanceResult): + """Record one performance run and upload it straight away. + + These come hours apart in a sweep, so they aren't held for a batch. + """ + self.payload.performance.append(result) + self._pending.performance.append(result) + self.flush() + + def flush(self): + """Upload whatever results are buffered.""" + if not (self._pending.results or self._pending.performance): + return + batch, self._pending = self._pending, ResultBatch() + if self._send( + "POST", + f"/runs/{self.payload.run.run_id}/results", + batch.model_dump(mode="json"), + what=f"{len(batch.results)} asset results and " + f"{len(batch.performance)} performance results", + ): + self.sent_results += len(batch.results) + self.sent_performance += len(batch.performance) + self.sent_batches += 1 + + def finish_run( + self, counts: Optional[dict] = None, finish: Optional[RunFinish] = None + ): + """Upload any remaining results and close the run. + + ``finish`` closes it with a given end time instead of now, for replays. + """ + self.flush() + self.payload.finish = finish or RunFinish( + ended_at=datetime.now().astimezone(), counts=counts or {} + ) + self._send( + "PATCH", + f"/runs/{self.payload.run.run_id}", + self.payload.finish.model_dump(mode="json"), + what="closing the run", + ) + if self._client is not None: + self._client.close() + self._client = None + + def save(self, output_dir: str, prefix: str = "") -> str: + """Write the whole run to ``output_dir`` for replay; return the path.""" + os.makedirs(output_dir, exist_ok=True) + path = os.path.join( + output_dir, f"{prefix}radiator_{self.payload.run.run_id}.json" + ) + with open(path, "w") as f: + f.write(self.payload.model_dump_json(indent=2)) + return path + + @property + def diff_url(self) -> Optional[str]: + """Link to what changed since the previous run, once a run is open.""" + return f"{self.run_url}/diff" if self.run_url else None + + def summary(self) -> Optional[dict]: + """The run against the previous one in its series (pass rates, + regressions, fixes), or None if it isn't available.""" + res = self._read("summary") + return res.json() if res is not None else None + + def history_png(self, runs: int = 30) -> Optional[bytes]: + """The series' pass-rate history up to this run, as a PNG, or None.""" + res = self._read("history.png", params={"runs": runs}) + return res.content if res is not None else None + + def performance_png(self, runs: int = 30) -> Optional[bytes]: + """Each of the run's services' max sustainable concurrency up to this + run, as a PNG, or None.""" + res = self._read("performance.png", params={"runs": runs}) + return res.content if res is not None else None + + def _read(self, what: str, params: Optional[dict] = None): + """GET something about this run from the read API. Best effort: only + once the whole run has been uploaded, and never raises.""" + if not self.enabled or self.upload_failed or self.payload is None: + return None + if self.payload.finish is None: + return None + return self._get( + f"/api/runs/{self.payload.run.run_id}/{what}", params, f"the run's {what}" + ) + + # --- run cycles and the weekly digest ------------------------------------ + + def cycle_url(self, cycle_id) -> Optional[str]: + return f"{self.public_url}/cycles/{cycle_id}" if self.enabled else None + + @property + def weekly_url(self) -> Optional[str]: + return f"{self.public_url}/weekly" if self.enabled else None + + def cycle(self, cycle_id) -> Optional[dict]: + """A run cycle's board (each test against its previous run, changes by + agent, flags), or None if it isn't available.""" + res = self._get(f"/api/cycles/{cycle_id}", what="the run cycle") + return res.json() if res is not None else None + + def cycle_board_png(self, cycle_id) -> Optional[bytes]: + res = self._get(f"/api/cycles/{cycle_id}/board.png", what="the cycle board") + return res.content if res is not None else None + + def digest(self, days: int = 7) -> Optional[dict]: + """Every environment's latest runs over the last ``days``, or None.""" + res = self._get("/api/digest", {"days": days}, "the weekly digest") + return res.json() if res is not None else None + + def digest_png(self, days: int = 7) -> Optional[bytes]: + res = self._get("/api/digest.png", {"days": days}, "the weekly digest") + return res.content if res is not None else None + + def _get(self, path: str, params: Optional[dict] = None, what: str = "it"): + """GET from the read API. Best effort: never raises.""" + if not self.enabled: + return None + try: + res = self._http().get(f"{self.base_url}{path}", params=params) + res.raise_for_status() + return res + except Exception as e: + self.logger.warning( + f"Couldn't get {what} from the Information Radiator: {e}" + ) + return None + + def _http(self) -> httpx.Client: + if self._client is None: + self._client = httpx.Client( + base_url=f"{self.base_url}{INGEST_PREFIX}", + headers={"Authorization": f"Bearer {self.token}"}, + timeout=30, + # retries connection failures only; an HTTP error is an answer + transport=httpx.HTTPTransport(retries=2), + ) + return self._client + + def _send(self, method: str, path: str, body: dict, what: str = "") -> bool: + """Send one write. Returns True if the radiator took it. + + The first failure stops the rest of the run's uploads (see + ``upload_failed``), so it logs everything needed to see why: the + radiator's own answer (eg which field a 422 rejected), or the + connection error. + """ + if not self.enabled or self.upload_failed: + return False + try: + res = self._http().request(method, path, json=body) + res.raise_for_status() + return True + except httpx.HTTPStatusError as e: + reason = ( + f"the radiator answered {e.response.status_code}: " + f"{e.response.text[:2000]}" + ) + except Exception as e: + reason = f"{type(e).__name__}: {e}" + self.upload_failed = True + run_id = self.payload.run.run_id if self.payload else "?" + self.logger.error( + f"Upload to the Information Radiator failed ({method} " + f"{self.base_url}{INGEST_PREFIX}{path}" + f"{', ' + what if what else ''}): {reason}. Not sending the rest of " + f"run {run_id}; the whole run will be saved locally for " + "`test-harness-radiator push`." + ) + return False + + def outcome(self, expected: list) -> List[str]: + """Log lines on what the radiator got, against ``expected`` + (test case id, asset id) pairs: for the end of a run.""" + lines = [] + if self.payload is None: + return lines + recorded = {(r.test_case_id, r.asset_id) for r in self.payload.results} + recorded |= {(r.test_case_id, r.asset_id) for r in self.payload.performance} + have = len(recorded & set(expected)) + perf = len(self.payload.performance) + perf_note = f" ({perf} performance results)" if perf else "" + if self.enabled and not self.upload_failed: + lines.append( + f"Information Radiator: {self.run_url} has {have} of the " + f"{len(expected)} assets in the suite{perf_note}, sent in " + f"{self.sent_batches} batches." + ) + elif self.enabled: + lines.append( + "Information Radiator: the upload failed partway (see the error " + f"above); it got {self.sent_results} asset results and " + f"{self.sent_performance} performance results before that. " + f"{have} of the {len(expected)} assets were recorded and saved " + "locally (below)." + ) + else: + lines.append( + f"Information Radiator: not uploaded; {have} of the " + f"{len(expected)} assets were recorded and saved locally (below)." + ) + missing = [pair for pair in expected if pair not in recorded] + if missing: + shown = ", ".join(f"{t}/{a}" for t, a in missing[:25]) + more = f" and {len(missing) - 25} more" if len(missing) > 25 else "" + lines.append( + f"{len(missing)} assets in the suite have no result for the " + f"Information Radiator: {shown}{more}. Search this log for an " + "asset id to see why." + ) + return lines + + +def push(payload: RunPayload, client: RadiatorClient) -> bool: + """Replay a saved run into the radiator. Returns True on success.""" + client.start_run(payload.run) + for result in payload.results: + client.add_result(result) + for result in payload.performance: + client.add_performance(result) + if payload.finish is not None: + # keep the original end time and counts, not the time of the replay + client.finish_run(finish=payload.finish) + else: + # the run never finished (eg the harness crashed); leave it open + client.flush() + return not client.upload_failed + + +def cli(argv=None): + """Entrypoint for ``test-harness-radiator``.""" + # where the radiator is: on each subcommand, so they go after it + common = ArgumentParser(add_help=False) + common.add_argument("--radiator_url", help="Defaults to $RADIATOR_URL") + common.add_argument("--radiator_token", help="Defaults to $RADIATOR_TOKEN") + parser = ArgumentParser(description="Information Radiator tools") + subparsers = parser.add_subparsers(dest="command", required=True) + push_parser = subparsers.add_parser( + "push", + parents=[common], + help="Upload a run saved by the harness (radiator_.json)", + ) + push_parser.add_argument("files", nargs="+", help="Saved run payload(s)") + digest_parser = subparsers.add_parser( + "digest", + parents=[common], + help=( + "Post the weekly digest to Slack: every environment's latest runs " + "of each test, with flags (to RADIATOR_SLACK_*, or SLACK_* when " + "Zebrunner isn't configured)" + ), + ) + digest_parser.add_argument( + "--days", type=int, default=7, help="How far back to look (default 7)" + ) + digest_parser.add_argument( + "--output_dir", + default="test_results", + help="Where to save the digest if Slack isn't configured", + ) + args = parser.parse_args(argv) + + logging.basicConfig(level=logging.INFO) + logger = logging.getLogger("test-harness-radiator") + if not RadiatorClient.is_configured(args.radiator_url, args.radiator_token): + logger.error("Set RADIATOR_URL and RADIATOR_TOKEN (or pass the flags).") + sys.exit(1) + + if args.command == "digest": + from test_harness.cycle import digest_cli + + client = RadiatorClient(args.radiator_url, args.radiator_token, logger=logger) + sys.exit(digest_cli(client, args.days, args.output_dir, logger)) + + failed = 0 + for path in args.files: + with open(path) as f: + payload = RunPayload.model_validate(json.load(f)) + client = RadiatorClient(args.radiator_url, args.radiator_token, logger=logger) + if push(payload, client): + logger.info(f"Pushed {path} -> {client.run_url}") + else: + failed += 1 + sys.exit(1 if failed else 0) diff --git a/test_harness/reporter.py b/test_harness/reporter.py index 2a0bdc9..0c35ade 100644 --- a/test_harness/reporter.py +++ b/test_harness/reporter.py @@ -18,6 +18,9 @@ class Reporter: """Reports tests and statuses to the Information Radiator.""" + # True for the stand-in that reports nowhere + is_local = False + def __init__( self, base_url=None, @@ -142,7 +145,6 @@ def create_test( ] ) else: - print("made it to the error section") raise Exception res = self.authenticated_client.post( url=f"{self.base_path}/api/reporting/v1/test-runs/{self.test_run_id}/tests", @@ -264,6 +266,8 @@ class LocalReporter(Reporter): uploading, so the rest of the harness is oblivious to the difference. """ + is_local = True + def __init__( self, base_url=None, diff --git a/test_harness/result_collector.py b/test_harness/result_collector.py index 6c22d5c..b5e75d1 100644 --- a/test_harness/result_collector.py +++ b/test_harness/result_collector.py @@ -14,7 +14,9 @@ TestEnvEnum, ) +from radiator_schema import AgentResult, AssetResult, PerformanceResult from test_harness.performance_test_runner import checkpoint_verdicts +from test_harness.radiator_client import RadiatorClient from test_harness.utils import AgentReport, AgentStatus, TestReport @@ -39,6 +41,7 @@ def __init__( test_env: Optional[TestEnvEnum], logger: logging.Logger, target: Optional[str] = None, + radiator: Optional[RadiatorClient] = None, ): """Initialize the Collector. @@ -47,8 +50,12 @@ def __init__( service directly, so results are collected for that one agent instead of the ARS + ARA roster; an ARS override still fans out to ARAs and keeps the usual roster. + + ``radiator``, when given, is handed a structured record of every result + as it is collected. """ self.logger = logger + self.radiator = radiator self.has_acceptance_results = False self.has_performance_results = False agents = [ @@ -104,6 +111,7 @@ def collect_acceptance_result( parent_pk: Union[str, None], url: str, force_skipped: bool = False, + status: Optional[AgentStatus] = None, ): """Add a single report to the total output. @@ -111,6 +119,10 @@ def collect_acceptance_result( in ``report``. It is used when the test as a whole was skipped (eg the query never ran) so the per-agent stats/CSV agree with the skipped test-level status instead of reporting incidental per-ARA errors. + + ``status`` is the test's overall status, for the radiator. Without it, + it is worked out the way ``run_tests`` does: from the ARS, or from the + single agent when an override target narrowed the roster to one. """ self.has_acceptance_results = True # add result to stats @@ -145,23 +157,131 @@ def collect_acceptance_result( f""""{asset.name}",{url},{pk_url},{test.id},{asset.id},{agent_results}\n""" ) - @staticmethod - def _expected_answer_cells(agent_report: AgentReport) -> List[str]: + if self.radiator is not None: + if status is None: + status_agent = "ars" if "ars" in self.agents else self.agents[0] + agent_report = report.result.get(status_agent) + status = ( + agent_report.status + if agent_report is not None and not force_skipped + else AgentStatus.SKIPPED + ) + # A record the schema rejects costs the radiator one asset, never + # the run its results. + try: + record = self._asset_result( + test, asset, report, parent_pk, status, force_skipped + ) + except Exception as e: + self.logger.error( + f"[{test.id}] couldn't record asset {asset.id} for the " + f"Information Radiator: {e}" + ) + else: + self.radiator.add_result(record) + + def _asset_result( + self, + test: Union[TestCase, PathfinderTestCase], + asset: Union[TestAsset, PathfinderTestAsset], + report: TestReport, + parent_pk: Optional[str], + status: AgentStatus, + force_skipped: bool, + ) -> AssetResult: + """The radiator's record of one asset, with a row for every agent. + + Agents are the same roster as the CSV's columns, and an agent with no + response is SKIPPED here too, so the two always agree. + """ + agents = [] + for agent in self.agents: + pk = report.pks.get(agent) + # the query runner writes the string "None" when the ARS never + # produced a merged message + pk = None if pk in (None, "", "None") else pk + agent_report = None if force_skipped else report.result.get(agent) + if agent_report is None: + agents.append( + AgentResult(agent=agent, status=AgentStatus.SKIPPED.value, pk=pk) + ) + continue + found, rank, score = self._expected_answer(agent_report) + agents.append( + AgentResult( + agent=agent, + status=agent_report.status.value, + message=agent_report.message, + http_status=agent_report.http_status, + found=found, + rank=rank, + score=score, + n_results=agent_report.n_results, + response_time_s=agent_report.response_time_s, + pk=pk, + expected_nodes_found=getattr( + agent_report, "expected_nodes_found", None + ), + ) + ) + + if isinstance(asset, PathfinderTestAsset): + kind = "pathfinder" + input_curie = asset.source_input_id + output_curie = asset.target_input_id + else: + kind = "acceptance" + input_curie = asset.input_id + output_curie = asset.output_id + return AssetResult( + test_case_id=test.id, + asset_id=asset.id, + kind=kind, + name=asset.name or asset.description, + expected_output=asset.expected_output, + predicate=asset.predicate_id, + input_curie=input_curie, + output_curie=output_curie, + status=status.value, + parent_pk=parent_pk or None, + agents=agents, + details=report.test_details or {}, + ) + + @classmethod + def _expected_answer_cells(cls, agent_report: AgentReport) -> List[str]: """CSV cells for where the expected answer landed for a single agent. + Returns (found, rank, score) as strings; see ``_expected_answer``. + Cells are left blank when the agent never got far enough for the + question to have an answer (eg it errored out). + """ + found, rank, score = cls._expected_answer(agent_report) + return [ + "" if found is None else ("true" if found else "false"), + "" if rank is None else str(rank), + "" if score is None else str(score), + ] + + @staticmethod + def _expected_answer( + agent_report: AgentReport, + ) -> Tuple[Optional[bool], Optional[int], Optional[float]]: + """Where the expected answer landed for a single agent. + Returns (found, rank, score), pulled from the same ``actual_output`` that goes into the report JSON uploaded to the radiator. ARS results are scored/ranked by the ARS confidence score, ARA results by their own analyses, so whichever of the two the analysis filled in is used. - Cells are left blank when the agent never got far enough for the - question to have an answer (eg it errored out). + Each is None when the agent never got far enough for the question to + have an answer (eg it errored out). """ actual_output = agent_report.actual_output or {} if not actual_output: # No results at all means the expected answer definitely wasn't in # the response; anything else (an error, a timeout) is unknown. - found = "false" if agent_report.status == AgentStatus.NO_RESULTS else "" - return [found, "", ""] + found = False if agent_report.status == AgentStatus.NO_RESULTS else None + return found, None, None rank = actual_output.get("ars_rank") if rank is None: @@ -175,11 +295,7 @@ def _expected_answer_cells(agent_report: AgentReport) -> List[str]: # answer that was in the response. found = rank is not None or score is not None - return [ - "true" if found else "false", - "" if rank is None else str(rank), - "" if score is None else str(score), - ] + return bool(found), rank, score def collect_performance_result( self, @@ -250,6 +366,69 @@ def collect_performance_result( "summary": summary, } + if self.radiator is not None: + # same verdict run_tests gives the test: a run with no summary, or + # one that missed a checkpoint, failed + failed = bool(results.get("error")) or ( + summary.get("checkpoints_passed") is False + ) + try: + record = PerformanceResult( + test_case_id=test.id, + asset_id=asset.id, + host=host_url, + helmsdeep_target=results.get("helmsdeep_target"), + profile=results.get("profile"), + status=( + AgentStatus.FAILED.value if failed else AgentStatus.PASSED.value + ), + error=results.get("error"), + exit_code=results.get("exit_code"), + max_sustainable_concurrency=summary.get( + "max_sustainable_concurrency" + ), + knee_unsupported=summary.get("knee_unsupported"), + checkpoints_passed=summary.get("checkpoints_passed"), + summary=summary, + ) + except Exception as e: + self.logger.error( + f"[{test.id}] couldn't record the performance run against " + f"{host_url} for the Information Radiator: {e}" + ) + else: + self.radiator.add_performance(record) + + def flush(self): + """Upload whatever the radiator has buffered.""" + if self.radiator is not None: + self.radiator.flush() + + def record_performance_error( + self, + test: Union[TestCase, PathfinderTestCase], + asset: Union[TestAsset, PathfinderTestAsset], + host_url: str, + error: str, + ): + """Record, for the radiator, a performance run that raised outright. + + A run that returned an error goes through ``collect_performance_result``; + this is for one that never returned at all, which would otherwise leave + no trace of the attempt in the radiator. + """ + if self.radiator is None: + return + self.radiator.add_performance( + PerformanceResult( + test_case_id=test.id, + asset_id=asset.id, + host=host_url, + status=AgentStatus.FAILED.value, + error=error, + ) + ) + @property def performance_checkpoints_passed(self) -> Optional[bool]: """Overall checkpoint verdict across every performance run. diff --git a/test_harness/run.py b/test_harness/run.py index a86d50c..1634f2b 100644 --- a/test_harness/run.py +++ b/test_harness/run.py @@ -25,9 +25,10 @@ resolve_profile, run_performance_test, ) -from test_harness.reporter import Reporter +from test_harness.reporter import LocalReporter, Reporter from test_harness.result_collector import ResultCollector from test_harness.runner.query_runner import QueryRunner, env_map +from test_harness.trapi import DEFAULT_TRAPI_VERSION from test_harness.utils import ( AgentReport, AgentStatus, @@ -36,6 +37,27 @@ ) +def record_response_meta(agent_report: AgentReport, response: Any) -> None: + """Record what the response itself says, before any analysis of it. + + ``n_results`` is only recorded for a successful response: the query runner + stands in an empty result list when an agent errors or times out, and that + is not an agent answering with zero results. + """ + if not isinstance(response, dict): + return + status_code = response.get("status_code") + # after a failed ARS poll this is the ARS's status string, eg "Running" + agent_report.http_status = status_code if isinstance(status_code, int) else None + agent_report.response_time_s = response.get("elapsed_s") + body = response.get("response") + message = body.get("message") if isinstance(body, dict) else None + results = message.get("results") if isinstance(message, dict) else None + status_code = agent_report.http_status + if isinstance(results, list) and status_code is not None and status_code < 300: + agent_report.n_results = len(results) + + def run_tests( tests: Dict[str, Union[TestCase, PathfinderTestCase]], reporter: Reporter, @@ -47,9 +69,16 @@ def run_tests( logger.info(f"Running {len(tests)} queries...") target_url = args.get("target_url") target = args.get("target") - query_runner = QueryRunner(logger, target_url=target_url, target=target) + trapi_version = args.get("trapi_version") or DEFAULT_TRAPI_VERSION + query_runner = QueryRunner( + logger, + target_url=target_url, + target=target, + trapi_version=trapi_version, + query_parameters=args.get("query_parameters"), + ) logger.info("Runner is getting service registry") - query_runner.retrieve_registry(trapi_version=args["trapi_version"]) + query_runner.retrieve_registry(trapi_version=trapi_version) # The overall test status is normally driven by the ARS. When the target # service specified in the tests is overridden (eg to run against a # locally running ARA), it is driven by the override target instead. @@ -60,7 +89,11 @@ def run_tests( for test in tqdm(list(tests.values())): # check if acceptance test if not test.test_assets or not test.test_case_objective: - logger.warning(f"Test has missing required fields: {test.id}") + missing = "test_assets" if not test.test_assets else "test_case_objective" + logger.warning( + f"[{test.id}] skipped, and not sent to the Information Radiator: " + f"the test case has no {missing}" + ) continue query_responses = {} @@ -72,17 +105,23 @@ def run_tests( # throw out any assets with unsupported expected outputs, i.e. OverlyGeneric if asset.expected_output not in collector.query_types: logger.warning( - f"Asset id {asset.id} has unsupported expected output." + f"[{test.id}] asset {asset.id} skipped, and not sent to " + "the Information Radiator: its expected output " + f"{asset.expected_output!r} isn't one the harness tests " + f"({', '.join(collector.query_types)})" ) continue # create test in Test Dashboard test_id = "" + asset_reporter = reporter try: test_id = reporter.create_test(test, asset) test_ids.append(test_id) except Exception: + # Zebrunner failing for this asset mustn't drop it: run + # and collect it anyway, just without reporting it there. logger.error(f"Failed to create test: {test.id}") - continue + asset_reporter = LocalReporter(logger=logger) test_asset_hash = hash_test_asset(asset) test_query = query_responses.get(test_asset_hash) @@ -90,7 +129,7 @@ def run_tests( message = json.dumps(test_query["query"], indent=4) else: message = "Unable to retrieve response for test asset." - reporter.upload_log( + asset_reporter.upload_log( test_id, message, ) @@ -125,6 +164,7 @@ def run_tests( actual_output=None, ) agent_report = report.result[agent] + record_response_meta(agent_report, response) try: if response.get("submission_failed"): # The query never made it into the component @@ -226,6 +266,7 @@ def run_tests( test_query["pks"].get("parent_pk"), f"{reporter.base_path}/test-runs/{reporter.test_run_id}/tests/{test_id}", force_skipped=force_skipped, + status=status, ) try: @@ -260,16 +301,22 @@ def run_tests( for ara in collector.agents if ara in report.result ] - reporter.upload_labels(test_id, labels) + asset_reporter.upload_labels(test_id, labels) except Exception as e: logger.warning(f"[{test.id}] failed to upload labels: {e}") - logger.info(f"Full report: {json.dumps(asdict(report), indent=4)}") - reporter.upload_log(test_id, json.dumps(asdict(report), indent=4)) + asset_reporter.upload_log( + test_id, json.dumps(asdict(report), indent=4) + ) else: # No query response for this asset (eg query generation # failed). Record it as skipped across every agent so it # still appears in the per-agent stats, CSV, and radiator # labels as SKIPPED instead of being dropped entirely. + logger.warning( + f"[{test.id}] asset {asset.id}: no query response (its " + "query couldn't be built or sent; see the errors above), " + "so it's recorded as SKIPPED" + ) status = AgentStatus.SKIPPED collector.collect_acceptance_result( test, @@ -278,9 +325,10 @@ def run_tests( None, f"{reporter.base_path}/test-runs/{reporter.test_run_id}/tests/{test_id}", force_skipped=True, + status=status, ) try: - reporter.upload_labels( + asset_reporter.upload_labels( test_id, [ {"key": ara, "value": AgentStatus.SKIPPED.value} @@ -290,20 +338,27 @@ def run_tests( except Exception as e: logger.warning(f"[{test.id}] failed to upload labels: {e}") - reporter.finish_test(test_id, status.value) + asset_reporter.finish_test(test_id, status.value) collector.acceptance_report[status.value] += 1 elif test.test_case_objective == "QuantitativeTest": # create test in Test Dashboard test_ids = [] for asset in test.test_assets: test_id = "" + asset_reporter = reporter try: test_id = reporter.create_test(test, asset) test_ids.append(test_id) except Exception as e: - logger.error(f"Failed to create test: {test.id}", e) - continue + logger.error(f"Failed to create test: {test.id}: {e}") + asset_reporter = LocalReporter(logger=logger) + if not isinstance(test, PerformanceTestCase): + logger.warning( + f"[{test.id}] asset {asset.id} skipped, and not sent to " + "the Information Radiator: only performance tests are " + "run as QuantitativeTests" + ) if isinstance(test, PerformanceTestCase): if target_url is not None: host = query_runner.target_url @@ -324,7 +379,7 @@ def run_tests( run_type = helmsdeep_target( perf_target or test.components[0], profile ) - reporter.upload_log( + asset_reporter.upload_log( test_id, describe_run(test, host, run_type, profile), ) @@ -370,10 +425,12 @@ def run_tests( status = AgentStatus.FAILED except Exception as e: logger.error( - f"Failed to run performance test for {test.id}: {e}" + f"[{test.id}] performance test against {host} failed: " + f"{type(e).__name__}: {e}" ) status = AgentStatus.FAILED - reporter.finish_test(test_id, status.value) + collector.record_performance_error(test, asset, host, str(e)) + asset_reporter.finish_test(test_id, status.value) # try: # test_inputs = [ # assets.id, @@ -410,7 +467,10 @@ def run_tests( else: try: test_id = reporter.create_test(test, test.test_assets[0]) - logger.error(f"Unsupported test type: {test.id}") + logger.error( + f"[{test.id}] not run, and not sent to the Information " + f"Radiator: unsupported test type {test.test_case_objective!r}" + ) reporter.upload_log( test_id, f"Unsupported test type in test: {test.id}" ) @@ -419,5 +479,10 @@ def run_tests( except Exception: logger.error(f"Failed to report errors with: {test.id}") + # Send this test case's results to the radiator now rather than when a + # batch fills: each case waits minutes on its queries, so a whole run + # can be under one batch, and nothing would show until the end. + collector.flush() + # delete this big object to help out the garbage collector del query_responses diff --git a/test_harness/runner/generate_query.py b/test_harness/runner/generate_query.py index c9af93a..e7923ce 100644 --- a/test_harness/runner/generate_query.py +++ b/test_harness/runner/generate_query.py @@ -1,153 +1,194 @@ -"""Given a Test Asset, generate a TRAPI query.""" +"""Given a Test Asset, generate a TRAPI query. -import copy -from typing import Union +Queries are built from ``translator_tom`` (TOM) models for the TRAPI version +under test, so a malformed query fails here, against that version's schema, +rather than at the service it was sent to. ``to_dict()`` drops unset fields, +which is the JSON that goes on the wire. +""" +from typing import Dict, List, Optional, Union + +import translator_tom.v1_6 as trapi_1_6 +import translator_tom.v2_0 as trapi_2_0 from translator_testing_model.datamodel.pydanticmodel import ( PathfinderTestAsset, TestAsset, ) +from test_harness.trapi import ( + DEFAULT_TRAPI_VERSION, + apply_query_parameters, + is_trapi_2, +) from test_harness.utils import QUERY_TYPE_PREDICATES, get_qualifier_constraints -MVP1 = { - "message": { - "query_graph": { - "nodes": { - "ON": {"categories": ["biolink:Disease"]}, - "SN": {"categories": ["biolink:ChemicalEntity"]}, - }, - "edges": { - "t_edge": { - "object": "ON", - "subject": "SN", - "predicates": ["biolink:treats"], - } - }, - } - } +# Which query node a test asset's input pins, by query type and input +# category. SN is the subject (the chemical), ON the object. +INPUT_NODES = { + "MVP1": {"biolink:Disease": "ON"}, + "MVP2": {"biolink:ChemicalEntity": "SN", "biolink:Gene": "ON"}, } -MVP2 = { - "message": { - "query_graph": { - "nodes": { - "ON": {"categories": ["biolink:Gene"]}, - "SN": {"categories": ["biolink:ChemicalEntity"]}, - }, - "edges": { - "t_edge": { - "object": "ON", - "subject": "SN", - "predicates": ["biolink:affects"], - "qualifier_constraints": [ - { - "qualifier_set": [ - { - "qualifier_type_id": "biolink:object_aspect_qualifier", - "qualifier_value": "", - }, - { - "qualifier_type_id": "biolink:object_direction_qualifier", - "qualifier_value": "", - }, - ] - } - ], - } - }, - } - } +# The categories of the two query nodes, by query type. +NODE_CATEGORIES = { + "MVP1": {"SN": "biolink:ChemicalEntity", "ON": "biolink:Disease"}, + "MVP2": {"SN": "biolink:ChemicalEntity", "ON": "biolink:Gene"}, } -PATHFINDER = { - "message": { - "query_graph": { - "nodes": { - "SN": { - "set_interpretation": "BATCH", - "constraints": [], - "member_ids": [], - }, - "ON": { - "set_interpretation": "BATCH", - "constraints": [], - "member_ids": [], - }, - }, - "paths": {"p0": {"subject": "SN", "object": "ON"}}, - } + +def _query_type(test_asset: TestAsset) -> str: + for query_type, predicate in QUERY_TYPE_PREDICATES.items(): + if test_asset.predicate_id == predicate: + return query_type + raise Exception(f"Unsupported predicate: {test_asset.predicate_id}") + + +def _qualifier_set(test_asset: TestAsset) -> Dict[str, str]: + """The asset's object aspect / direction qualifiers, leaving out unset ones.""" + aspect_qualifier, direction_qualifier = get_qualifier_constraints(test_asset) + qualifiers = { + "biolink:object_aspect_qualifier": aspect_qualifier, + "biolink:object_direction_qualifier": direction_qualifier, } -} + return {type_id: value for type_id, value in qualifiers.items() if value} -def generate_query(test_asset: Union[TestAsset, PathfinderTestAsset]) -> dict: - """Generate a TRAPI query.""" - query = {} - if isinstance(test_asset, PathfinderTestAsset): - source_id = test_asset.source_input_id - target_id = test_asset.target_input_id - query = copy.deepcopy(PATHFINDER) - query["message"]["query_graph"]["nodes"]["SN"] = { - "ids": [source_id], - "categories": [test_asset.source_input_category], - } - query["message"]["query_graph"]["nodes"]["ON"] = { - "ids": [target_id], - "categories": [test_asset.target_input_category], - } - elif test_asset.predicate_id == QUERY_TYPE_PREDICATES["MVP1"]: - # MVP1 - query = copy.deepcopy(MVP1) - # add id to node - if test_asset.input_category == "biolink:Disease": - query["message"]["query_graph"]["nodes"]["ON"]["ids"] = [ - test_asset.input_id - ] - else: - raise Exception( - f"Unsupported input category for MVP1: {test_asset.input_category}" - ) - # add knowledge_type - if "inferred" in test_asset.test_runner_settings: - query["message"]["query_graph"]["edges"]["t_edge"][ - "knowledge_type" - ] = "inferred" - elif test_asset.predicate_id == QUERY_TYPE_PREDICATES["MVP2"]: - # MVP2 - query = copy.deepcopy(MVP2) - # add id to corresponding node - if test_asset.input_category == "biolink:ChemicalEntity": - query["message"]["query_graph"]["nodes"]["SN"]["ids"] = [ - test_asset.input_id - ] - elif test_asset.input_category == "biolink:Gene": - query["message"]["query_graph"]["nodes"]["ON"]["ids"] = [ - test_asset.input_id +def _categories(category: Optional[str]) -> Optional[List[str]]: + return [category] if category else None + + +def _one_hop_query_2_0( + query_type: str, test_asset: TestAsset, input_node: str +) -> trapi_2_0.Query: + qualifier_set = _qualifier_set(test_asset) if query_type == "MVP2" else {} + edge = trapi_2_0.QEdge( + subject="SN", + object="ON", + predicates=[test_asset.predicate_id], + # TRAPI 2.0 gathers every edge constraint into one object, and a + # QualifierSetConstraint is a plain {qualifier_type_id: value} mapping. + constraints=( + trapi_2_0.QEdgeConstraints(qualifiers=[qualifier_set]) + if qualifier_set + else None + ), + knowledge_type=( + "inferred" if "inferred" in test_asset.test_runner_settings else None + ), + ) + nodes = { + node: trapi_2_0.QNode( + categories=[category], + ids=[test_asset.input_id] if node == input_node else None, + ) + for node, category in NODE_CATEGORIES[query_type].items() + } + return trapi_2_0.Query( + message=trapi_2_0.Message( + query_graph=trapi_2_0.QueryGraph(nodes=nodes, edges={"t_edge": edge}) + ) + ) + + +def _one_hop_query_1_6( + query_type: str, test_asset: TestAsset, input_node: str +) -> trapi_1_6.Query: + qualifier_set = _qualifier_set(test_asset) if query_type == "MVP2" else {} + edge = trapi_1_6.QEdge( + subject="SN", + object="ON", + predicates=[test_asset.predicate_id], + qualifier_constraints=( + [ + trapi_1_6.QualifierConstraint( + qualifier_set=[ + trapi_1_6.Qualifier( + qualifier_type_id=type_id, qualifier_value=value + ) + for type_id, value in qualifier_set.items() + ] + ) ] - else: - raise Exception(f"Unsupported input category: {test_asset.input_category}") - # add qualifier constraints - aspect_qualifier, direction_qualifier = get_qualifier_constraints(test_asset) - query["message"]["query_graph"]["edges"]["t_edge"]["qualifier_constraints"][0][ - "qualifier_set" - ][0]["qualifier_value"] = aspect_qualifier - query["message"]["query_graph"]["edges"]["t_edge"]["qualifier_constraints"][0][ - "qualifier_set" - ][1]["qualifier_value"] = direction_qualifier - # add knowledge_type - if "inferred" in test_asset.test_runner_settings: - query["message"]["query_graph"]["edges"]["t_edge"][ - "knowledge_type" - ] = "inferred" - else: - raise Exception(f"Unsupported predicate: {test_asset.predicate_id}") + if qualifier_set + else None + ), + knowledge_type=( + "inferred" if "inferred" in test_asset.test_runner_settings else None + ), + ) + nodes = { + node: trapi_1_6.QNode( + categories=[category], + ids=[test_asset.input_id] if node == input_node else None, + ) + for node, category in NODE_CATEGORIES[query_type].items() + } + return trapi_1_6.Query( + message=trapi_1_6.Message( + query_graph=trapi_1_6.QueryGraph(nodes=nodes, edges={"t_edge": edge}) + ) + ) + + +def _pathfinder_query( + test_asset: PathfinderTestAsset, trapi_version: str +) -> Union[trapi_2_0.Query, trapi_1_6.Query]: + trapi = trapi_2_0 if is_trapi_2(trapi_version) else trapi_1_6 + nodes = { + "SN": trapi.QNode( + ids=[test_asset.source_input_id], + categories=_categories(test_asset.source_input_category), + ), + "ON": trapi.QNode( + ids=[test_asset.target_input_id], + categories=_categories(test_asset.target_input_category), + ), + } + paths = {"p0": trapi.QPath(subject="SN", object="ON")} + # TRAPI 2.0 makes QueryGraph.edges optional so one model covers Pathfinder + # queries; 1.6 has a separate PathfinderQueryGraph for them. + query_graph = ( + trapi_2_0.QueryGraph(nodes=nodes, paths=paths) + if trapi is trapi_2_0 + else trapi_1_6.PathfinderQueryGraph(nodes=nodes, paths=paths) + ) + return trapi.Query(message=trapi.Message(query_graph=query_graph)) - return query + +def generate_query( + test_asset: Union[TestAsset, PathfinderTestAsset], + trapi_version: str = DEFAULT_TRAPI_VERSION, + query_parameters: Optional[Dict] = None, +) -> dict: + """Generate a TRAPI query for ``trapi_version`` (2.0.x or 1.6.x). + + ``query_parameters`` (already checked by ``validate_query_parameters``) + go out with the query: under ``parameters`` in TRAPI 2.0, as top-level + fields in 1.6. + """ + if isinstance(test_asset, PathfinderTestAsset): + return apply_query_parameters( + _pathfinder_query(test_asset, trapi_version).to_dict(), + query_parameters, + trapi_version, + ) + + query_type = _query_type(test_asset) + input_node = INPUT_NODES[query_type].get(test_asset.input_category) + if input_node is None: + raise Exception( + f"Unsupported input category for {query_type}: " + f"{test_asset.input_category}" + ) + if is_trapi_2(trapi_version): + query = _one_hop_query_2_0(query_type, test_asset, input_node) + else: + query = _one_hop_query_1_6(query_type, test_asset, input_node) + return apply_query_parameters(query.to_dict(), query_parameters, trapi_version) if __name__ == "__main__": - test_asset = TestAsset.parse_obj( + test_asset = TestAsset.model_validate( { "id": "Asset_450", "name": "NeverShow: MMP3 increases activity or abundance of Potassium ion", @@ -194,4 +235,3 @@ def generate_query(test_asset: Union[TestAsset, PathfinderTestAsset]) -> dict: } ) query = generate_query(test_asset) - print(query) diff --git a/test_harness/runner/query_runner.py b/test_harness/runner/query_runner.py index dab0a7a..f63b932 100644 --- a/test_harness/runner/query_runner.py +++ b/test_harness/runner/query_runner.py @@ -12,6 +12,7 @@ from test_harness.runner.generate_query import generate_query from test_harness.runner.smart_api_registry import retrieve_registry_from_smartapi +from test_harness.trapi import DEFAULT_TRAPI_VERSION, trapi_minor_version from test_harness.utils import hash_test_asset, normalize_curies MAX_QUERY_TIME = 600 @@ -33,9 +34,17 @@ def __init__( logger: logging.Logger, target_url: Optional[str] = None, target: Optional[str] = None, + trapi_version: str = DEFAULT_TRAPI_VERSION, + query_parameters: Optional[Dict] = None, ): """Initialize the Query Runner. + ``trapi_version`` is the TRAPI version queries are written in, and the + version of the services picked from the SmartAPI registry. + + ``query_parameters`` go out with every query (see + ``trapi.validate_query_parameters``). + ``target_url`` and ``target`` override the target service specified in the tests themselves: when given, every query is sent to ``target_url`` instead of the services registered for the test's components, and the @@ -49,6 +58,10 @@ def __init__( if target is not None and not target.startswith("infores:"): target = f"infores:{target}" self.target_infores = target + # fail on an unsupported version now, not once per generated query + trapi_minor_version(trapi_version) + self.trapi_version = trapi_version + self.query_parameters = query_parameters def retrieve_registry(self, trapi_version: str): if self.target_url is not None: @@ -82,14 +95,25 @@ def run_query( # send message response = {} status_code = 418 + elapsed_s = None with httpx.Client(timeout=600) as client: try: + start_time = time.monotonic() res = client.post(url, json=message) + elapsed_s = time.monotonic() - start_time status_code = res.status_code res.raise_for_status() response = res.json() + except httpx.HTTPStatusError as e: + self.logger.error( + f"{infores} at {url} answered {status_code}: " + f"{e.response.text[:500]}" + ) except Exception as e: - self.logger.error(f"Something went wrong: {e}") + # status_code stays 418, the harness's "no HTTP answer at all" + self.logger.error( + f"No answer from {infores} at {url}: {type(e).__name__}: {e}" + ) if infores == "infores:ars": parent_pk = response.get("pk", "") @@ -116,6 +140,9 @@ def run_query( responses[single_infores] = { "response": response, "status_code": status_code, + # Only a direct query is timed: the ARS's own submit returns + # straight away, and its ARAs are only seen through polling. + "elapsed_s": elapsed_s, } return query_hash, responses, pks @@ -154,7 +181,7 @@ def get_ars_child_response( # query errored, need to capture break elif status == "Running": - self.logger.info(f"{infores} is still Running...") + self.logger.debug(f"{infores} is still Running...") current_time = time.time() time.sleep(10) else: @@ -320,14 +347,19 @@ def run_queries( if asset_hash not in queries: # generate query try: - query = generate_query(test_asset) + query = generate_query( + test_asset, self.trapi_version, self.query_parameters + ) queries[asset_hash] = { "query": query, "responses": {}, "pks": {}, } except Exception as e: - self.logger.warning(e) + self.logger.error( + f"[{test_case.id}] couldn't build the query for asset " + f"{test_asset.id}: {type(e).__name__}: {e}" + ) if self.target_url is not None: # ignore the components specified in the test case and send every @@ -372,4 +404,7 @@ def _send_queries(self, services: List[Dict[str, str]], queries: Dict[int, dict] queries[query_hash]["responses"].update(responses) queries[query_hash]["pks"].update(pks) except Exception as e: - self.logger.error(f"Something went wrong with the queries: {e}") + self.logger.error( + "Sending the queries failed partway, so some have no response: " + f"{type(e).__name__}: {e}" + ) diff --git a/test_harness/runner/smart_api_registry.py b/test_harness/runner/smart_api_registry.py index 0a00c88..a3addb8 100644 --- a/test_harness/runner/smart_api_registry.py +++ b/test_harness/runner/smart_api_registry.py @@ -4,14 +4,48 @@ import logging import re from collections import defaultdict +from typing import List import httpx +from test_harness.trapi import DEFAULT_TRAPI_VERSION + LOGGER = logging.getLogger(__name__) +SMARTAPI_URL = "https://smart-api.info/api/query?limit=1000&q=TRAPI" + + +def registered_trapi_versions(infores: str, maturity: str, hits=None) -> List[str]: + """The TRAPI versions ``infores`` is registered with in SmartAPI for one + maturity (eg "staging"), newest first. ``hits`` are SmartAPI's + registrations, fetched when not given.""" + if hits is None: + with httpx.Client(timeout=30) as client: + response = client.get(SMARTAPI_URL) + response.raise_for_status() + hits = response.json().get("hits", []) + versions = set() + for hit in hits: + info = hit.get("info") or {} + if (info.get("x-translator") or {}).get("infores") != infores: + continue + version = (info.get("x-trapi") or {}).get("version") + if not version: + continue + if any( + server.get("x-maturity") == maturity for server in hit.get("servers") or [] + ): + versions.add(version) + + def key(version): + return tuple(int(part) for part in re.findall(r"\d+", version)[:3]) + + return sorted(versions, key=key, reverse=True) + + def retrieve_registry_from_smartapi( - target_trapi_version="1.6.0", + target_trapi_version=DEFAULT_TRAPI_VERSION, ): """Returns a dict of smart api service endpoints defined with a dict like { @@ -23,7 +57,7 @@ def retrieve_registry_from_smartapi( """ with httpx.Client(timeout=30) as client: try: - response = client.get("https://smart-api.info/api/query?limit=1000&q=TRAPI") + response = client.get(SMARTAPI_URL) response.raise_for_status() except httpx.HTTPError as e: LOGGER.error("Failed to query smart api. Exiting...") diff --git a/test_harness/slacker.py b/test_harness/slacker.py index 6b869e3..890e9c1 100644 --- a/test_harness/slacker.py +++ b/test_harness/slacker.py @@ -132,6 +132,31 @@ def _slugify_filename(name): return slug or "test_results" +# The new Information Radiator's own Slack channel, while it runs alongside +# Zebrunner. An incoming webhook posts to one channel only, so it needs its own; +# the bot token can be the same one (invite the bot to both channels). +def radiator_slacker(logger=None) -> "Slacker | None": + """A Slacker for the new radiator's own channel, from RADIATOR_SLACK_*, or + None when that isn't set up. Then SLACK_CHANNEL carries the radiator's + messages too, unless Zebrunner (ZE_*) is configured: that channel stays + Zebrunner's, and the radiator's messages aren't posted. + + RADIATOR_SLACK_TOKEN defaults to SLACK_TOKEN. + """ + url = os.getenv("RADIATOR_SLACK_WEBHOOK_URL") + channel = os.getenv("RADIATOR_SLACK_CHANNEL") + token = os.getenv("RADIATOR_SLACK_TOKEN") or os.getenv("SLACK_TOKEN") + if url and channel and token: + return Slacker(url=url, token=token, slack_channel=channel) + if url or channel or os.getenv("RADIATOR_SLACK_TOKEN"): + (logger or logging.getLogger(__name__)).warning( + "RADIATOR_SLACK_* is only partly set (it needs a webhook URL, a " + "channel, and a token or SLACK_TOKEN); ignoring it and falling " + "back to the default Slack routing." + ) + return None + + class LocalSlacker(Slacker): """A Slacker that writes results to disk instead of posting to Slack. diff --git a/test_harness/trapi.py b/test_harness/trapi.py new file mode 100644 index 0000000..03b63b1 --- /dev/null +++ b/test_harness/trapi.py @@ -0,0 +1,149 @@ +"""TRAPI version handling. + +The harness speaks TRAPI 2.0 by default and can still speak 1.6 for services +that have not moved yet. Queries are built with ``translator_tom`` (TOM), the +Translator-wide TRAPI object model, using the model package for the version +being tested, so a query is validated against that version's schema before it +is ever sent. + +Responses are read as plain dicts rather than parsed into TOM models: an ARS +merged message can be very large, and the harness only reads the bindings out +of it. What does differ between versions on that path is the shape of a +binding, which ``binding_ids`` reads either way. +""" + +import json +import re +from typing import Any, Dict, List, Optional + +import translator_tom.v2_0 as trapi_2_0 +from pydantic import ValidationError + +DEFAULT_TRAPI_VERSION = "2.0.0" +# --trapi_version auto: the version the environment's ARS (or the --target +# service) is registered with in SmartAPI, for while environments differ. +AUTO_TRAPI_VERSION = "auto" + +# The TRAPI minor versions there is a TOM model package for, keyed by the +# "major.minor" prefix of a SemVer version. +SUPPORTED_TRAPI_VERSIONS = ("2.0", "1.6") + + +def trapi_minor_version(trapi_version: str) -> str: + """Return the ``major.minor`` of a TRAPI SemVer version, if it is supported. + + Examples: + >>> trapi_minor_version("2.0.0") + '2.0' + >>> trapi_minor_version("1.6.0-beta") + '1.6' + """ + match = re.match(r"(\d+)\.(\d+)", trapi_version or "") + minor = f"{match.group(1)}.{match.group(2)}" if match else None + if minor not in SUPPORTED_TRAPI_VERSIONS: + raise ValueError( + f"Unsupported TRAPI version '{trapi_version}'. " + f"Expected one of: {', '.join(v + '.x' for v in SUPPORTED_TRAPI_VERSIONS)}." + ) + return minor + + +def is_trapi_2(trapi_version: str) -> bool: + """Whether ``trapi_version`` is a TRAPI 2.x version.""" + return trapi_minor_version(trapi_version).startswith("2.") + + +def binding_ids(binding: Any) -> List[str]: + """Return the ids bound by one node, edge or path binding. + + TRAPI 2.0 binds each QNode / QEdge / QPath to a single object carrying an + ``ids`` list; TRAPI 1.x binds it to a list of objects each carrying an + ``id``. Both are read here, since the ARS can hand back either. + + Examples: + >>> binding_ids({"ids": ["MONDO:1", "MONDO:2"]}) + ['MONDO:1', 'MONDO:2'] + >>> binding_ids([{"id": "MONDO:1"}, {"id": "MONDO:2", "attributes": []}]) + ['MONDO:1', 'MONDO:2'] + """ + if isinstance(binding, dict): + return [str(binding_id) for binding_id in binding.get("ids") or []] + return [str(entry["id"]) for entry in binding or [] if "id" in entry] + + +# --- query parameters ---------------------------------------------------------- + +# The query parameters TRAPI 1.6 has, as top-level fields of the Query. It has +# no "parameters" object, and no timeout. +TRAPI_1_6_PARAMETERS = ("log_level", "bypass_cache") + + +def parse_query_parameters(text: Optional[str]) -> Optional[Dict[str, Any]]: + """Read query parameters given on the command line or in the environment. + + ``text`` is a JSON object, eg ``{"timeout": 300, "bypass_cache": true}``, + or ``@path`` to a file holding one. Empty means none. + """ + if text is None or not text.strip(): + return None + text = text.strip() + if text.startswith("@"): + with open(text[1:]) as f: + text = f.read() + try: + params = json.loads(text) + except ValueError as e: + raise ValueError(f"Query parameters aren't valid JSON: {e}") from None + if not isinstance(params, dict): + raise ValueError('Query parameters must be a JSON object, eg {"timeout": 300}') + return params + + +def validate_query_parameters( + params: Optional[Dict[str, Any]], trapi_version: str +) -> Optional[Dict[str, Any]]: + """Check query parameters against the TRAPI version's schema. + + Returns them as they'll be sent, or raises ValueError saying what's wrong, + so a typo stops the run before any query goes out rather than every query + failing. TRAPI 2.0's ``parameters`` takes service-specific keys besides + the standard ones; TRAPI 1.6 only has ``log_level`` and ``bypass_cache``. + """ + if not params: + return None + # the standard parameters' types are the same in both versions + try: + checked = trapi_2_0.QueryParameters.model_validate(params) + except ValidationError as e: + problems = "; ".join( + f"{'.'.join(str(p) for p in err['loc'])}: {err['msg']}" + for err in e.errors() + ) + raise ValueError(f"Invalid query parameters: {problems}") from None + normalized = checked.model_dump(mode="json", exclude_unset=True) + if not is_trapi_2(trapi_version): + unsupported = sorted(set(normalized) - set(TRAPI_1_6_PARAMETERS)) + if unsupported: + raise ValueError( + f"TRAPI {trapi_minor_version(trapi_version)} queries can't carry " + f"{', '.join(unsupported)}: only " + f"{' and '.join(TRAPI_1_6_PARAMETERS)} exist before TRAPI 2.0" + ) + return normalized + + +def apply_query_parameters( + query: Dict[str, Any], params: Optional[Dict[str, Any]], trapi_version: str +) -> Dict[str, Any]: + """Add already-validated query parameters to a query dict. + + TRAPI 2.0 carries them in the query's ``parameters`` object; TRAPI 1.6 as + top-level fields of the query. + """ + if not params: + return query + if is_trapi_2(trapi_version): + query["parameters"] = {**(query.get("parameters") or {}), **params} + else: + query.update(params) + return query diff --git a/test_harness/utils.py b/test_harness/utils.py index 6185ee6..ff4a267 100644 --- a/test_harness/utils.py +++ b/test_harness/utils.py @@ -47,13 +47,19 @@ class AgentReport: # Where the expected answer landed in this agent's response: a "found" # flag plus the ARS/ARA rank (int) and score (float) when it was found. actual_output: Optional[dict[str, Union[bool, int, float, None]]] + # About the response itself rather than the analysis of it. None when it + # isn't known: no response, or (for response_time_s) one fanned out by the + # ARS, which the harness only sees through its 10s polling. + http_status: Optional[int] = None + n_results: Optional[int] = None + response_time_s: Optional[float] = None @dataclass class PathfinderReport(AgentReport): """Dictionary for single Pathfinder agent report.""" - expected_nodes_found: str + expected_nodes_found: Optional[str] = None @dataclass diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a12b28d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,27 @@ +"""Fixtures shared by every test.""" + +import pytest + +from test_harness.radiator_client import RadiatorClient + + +@pytest.fixture(autouse=True) +def isolate_radiator(monkeypatch, tmp_path): + """Keep the Information Radiator client out of the developer's environment. + + ``main`` saves the run for the radiator whenever it isn't uploaded, which + in tests is always. Saves aimed at the default ``test_results/`` land in + the test's tmp dir instead of the checkout, and a radiator configured in + the developer's shell is never used. + """ + monkeypatch.delenv("RADIATOR_URL", raising=False) + monkeypatch.delenv("RADIATOR_TOKEN", raising=False) + monkeypatch.delenv("RADIATOR_PUBLIC_URL", raising=False) + save = RadiatorClient.save + + def save_to_tmp(self, output_dir, prefix=""): + if output_dir == "test_results": + output_dir = str(tmp_path) + return save(self, output_dir, prefix) + + monkeypatch.setattr(RadiatorClient, "save", save_to_tmp) diff --git a/tests/helpers/mock_responses.py b/tests/helpers/mock_responses.py index 598de52..ed71715 100644 --- a/tests/helpers/mock_responses.py +++ b/tests/helpers/mock_responses.py @@ -1,4 +1,4 @@ -"""Mock Test Responses.""" +"""Mock Test Responses, in TRAPI 2.0.""" kp_response = { "message": { @@ -35,6 +35,8 @@ "subject": "MESH:D008687", "object": "MONDO:0005148", "predicate": "biolink:treats", + "knowledge_level": "knowledge_assertion", + "agent_type": "manual_agent", "sources": [ { "resource_id": "infores:kp0", @@ -48,30 +50,13 @@ "results": [ { "node_bindings": { - "n0": [ - { - "id": "MESH:D008687", - "attributes": [], - }, - ], - "n1": [ - { - "id": "MONDO:0005148", - "attributes": [], - }, - ], + "n0": {"ids": ["MESH:D008687"]}, + "n1": {"ids": ["MONDO:0005148"]}, }, "analyses": [ { - "resource_id": "kp0", - "edge_bindings": { - "n0n1": [ - { - "id": "n0n1", - "attributes": [], - }, - ], - }, + "resource_id": "infores:kp0", + "edge_bindings": {"n0n1": {"ids": ["n0n1"]}}, } ], }, diff --git a/tests/helpers/mocks.py b/tests/helpers/mocks.py index 2d7ba95..ad73399 100644 --- a/tests/helpers/mocks.py +++ b/tests/helpers/mocks.py @@ -86,9 +86,16 @@ def collect_acceptance_result( parent_pk: str | None, url: str, force_skipped: bool = False, + status=None, ): return super().collect_acceptance_result( - test, asset, report, parent_pk, url, force_skipped=force_skipped + test, + asset, + report, + parent_pk, + url, + force_skipped=force_skipped, + status=status, ) def collect_performance_result( diff --git a/tests/test_cycle.py b/tests/test_cycle.py new file mode 100644 index 0000000..e04282b --- /dev/null +++ b/tests/test_cycle.py @@ -0,0 +1,276 @@ +"""Tests for run cycles: acceptance, pathfinder and performance in one +environment, in that order, then the radiator's board in Slack.""" + +from unittest import mock + +import pytest + +from test_harness import cycle +from test_harness.cycle import Step, cli, cycle_message, digest_message, plan + +CYCLE_ID = "1b4e28ba-2fa1-11d2-883f-0016d3cca427" + + +def test_a_cycle_runs_acceptance_then_pathfinder_then_performance(): + steps = plan( + CYCLE_ID, + acceptance="sprint_4_tests", + pathfinder="pathfinder_tests", + performance="performance_tests", + performance_targets=[("ars", "https://ars.ci.transltr.io")], + performance_profile="mixed", + output_dir="out", + harness_args=["--log_level", "INFO"], + download=True, + ) + assert [s.name for s in steps] == [ + "acceptance (sprint_4_tests)", + "pathfinder (pathfinder_tests)", + "performance (ars)", + ] + acceptance, pathfinder, performance = (s.cmd for s in steps) + assert acceptance == [ + "test-harness", + "--output_dir", + "out/acceptance", + "--cycle_id", + CYCLE_ID, + "--log_level", + "INFO", + "download", + "sprint_4_tests", + ] + assert pathfinder[-2:] == ["download", "pathfinder_tests"] + # the performance step is a sweep step: the target, and the profile + assert performance[:5] == [ + "test-harness", + "--target_url", + "https://ars.ci.transltr.io", + "--target", + "ars", + ] + assert "--performance_profile" not in acceptance + assert performance[performance.index("--performance_profile") + 1] == "mixed" + assert performance[performance.index("--cycle_id") + 1] == CYCLE_ID + assert performance[-2:] == ["download", "performance_tests"] + + +def test_a_cycle_can_leave_steps_out(): + steps = plan(CYCLE_ID, acceptance="a", tests_dir="suites") + assert [s.name for s in steps] == ["acceptance (a)"] + assert steps[0].cmd[-3:] == ["a", "--tests_dir", "suites"] + + +def _ran(*codes): + return [Step(f"step {i}", [], code, 30.0) for i, code in enumerate(codes)] + + +@mock.patch("test_harness.cycle._slacker", return_value=mock.Mock()) +@mock.patch("test_harness.cycle.post_cycle_report") +@mock.patch("test_harness.cycle.subprocess.run") +def test_every_step_runs_even_when_one_fails(run, post, _slacker, monkeypatch): + monkeypatch.setenv("RADIATOR_URL", "http://radiator") + monkeypatch.setenv("RADIATOR_TOKEN", "tok") + run.side_effect = [mock.Mock(returncode=2), mock.Mock(returncode=0)] + code = cli(["--acceptance", "a", "--pathfinder", "p", "--download"]) + assert run.call_count == 2 + assert code == 1 + # the board is still posted, for the steps that ran + cycle_id, steps = post.call_args.args[:2] + assert [s.code for s in steps] == [2, 0] + for call in run.call_args_list: + cmd = call.args[0] + assert cmd[cmd.index("--cycle_id") + 1] == cycle_id + + +@mock.patch("test_harness.cycle.subprocess.run") +def test_without_the_radiator_there_is_no_board(run, monkeypatch): + monkeypatch.delenv("RADIATOR_URL", raising=False) + monkeypatch.delenv("RADIATOR_TOKEN", raising=False) + run.return_value = mock.Mock(returncode=0) + with mock.patch("test_harness.cycle.post_cycle_report") as post: + assert cli(["--acceptance", "a"]) == 0 + post.assert_not_called() + + +@pytest.mark.parametrize( + "argv, message", + [ + ([], "Nothing to run"), + (["--performance", "perf"], "needs a --performance_target"), + (["--acceptance", "a", "--", "--tests_url", "x"], "belongs before the `--`"), + (["--acceptance", "a", "--", "--cycle_id", "x"], "sets --cycle_id itself"), + ], +) +def test_bad_arguments(argv, message, capsys, monkeypatch): + monkeypatch.delenv("PERFORMANCE_TARGETS", raising=False) + with pytest.raises(SystemExit): + cli(argv) + assert message in capsys.readouterr().err + + +REPORT = { + "env": "ci", + "rows": [ + { + "label": "Acceptance", + "suite": "sprint_4_tests", + "host": None, + "value_text": "80.6%", + "unit": "pass rate", + "change_text": "▼2.8 pts", + "detail_text": "3 regressions · 1 fixed", + "verdict_label": "Regressed", + "flags": [ + {"text": "arax: no results on 5 assets that had results last run"} + ], + "changes": [ + { + "agent": "arax", + "regressions": [ + {"name": f"Asset {i}", "asset_id": f"A{i}"} for i in range(6) + ], + "fixed": [{"name": None, "asset_id": "A9"}], + } + ], + }, + { + "label": "Performance · ars", + "suite": "performance_tests", + "host": "https://ars.ci.transltr.io", + "value_text": "14.5", + "unit": "max concurrency", + "change_text": "▲0.9 (+7%)", + "detail_text": "checkpoints passed", + "verdict_label": "Steady", + "flags": [], + "changes": [], + }, + ], +} + + +def test_the_cycle_message(): + message = cycle_message(REPORT, _ran(0, 2, 0), "https://r/cycles/x") + assert message.startswith("*CI run cycle finished* · 2 of 3 steps ran · 1 h 30 min") + assert "• step 1: couldn't run (exit 2)" in message + assert ( + "• *Acceptance (sprint_4_tests)*: 80.6% pass rate, ▼2.8 pts · " + "3 regressions · 1 fixed · *Regressed*" in message + ) + assert "• *Performance · ars*: 14.5 max concurrency" in message + assert ( + "arax: 6 regressions (Asset 0, Asset 1, Asset 2, Asset 3 and 2 more); " + "1 fixed (A9)" in message + ) + assert "*⚑ Flags*\n• arax: no results on 5 assets" in message + assert message.endswith( + "" + ) + + +def test_the_digest_message(): + digest = { + "since": "2026-10-05T12:00:00+00:00", + "until": "2026-10-12T12:00:00+00:00", + "envs": [{"env": "ci", "rows": REPORT["rows"]}], + } + message = digest_message(digest, "https://r/weekly") + assert message.startswith("*Weekly test status* · 2026-10-05 to 2026-10-12") + assert "*CI* · ⚑ 1\n Acceptance: 80.6%, ▼2.8 pts · Regressed" in message + assert "• CI: arax: no results on 5 assets" in message + empty = digest_message({**digest, "envs": []}, None) + assert "No finished runs this week." in empty + + +class _Slack: + def __init__(self): + self.messages, self.files = [], [] + + def post_notification(self, messages=[]): + self.messages.extend(messages) + + def upload_binary_file(self, filename, content, initial_comment=None, title=None): + self.files.append(filename) + + +def test_posting_the_board(): + radiator = mock.Mock() + radiator.cycle.return_value = REPORT + radiator.cycle_board_png.return_value = b"\x89PNG" + radiator.cycle_url.return_value = "https://r/cycles/x" + slack = _Slack() + assert cycle.post_cycle_report("x", _ran(0), radiator, slack, mock.Mock()) + assert "*CI run cycle finished*" in slack.messages[0] + assert slack.files == ["ci_run_cycle.png"] + + radiator.cycle.return_value = None + assert not cycle.post_cycle_report("x", _ran(0), radiator, _Slack(), mock.Mock()) + + +def test_posting_the_digest(): + radiator = mock.Mock() + radiator.digest.return_value = { + "since": "2026-10-05", + "until": "2026-10-12", + "envs": [], + } + radiator.digest_png.return_value = b"\x89PNG" + slack = _Slack() + assert cycle.post_digest(radiator, slack, mock.Mock()) + assert slack.files == ["weekly_status.png"] + radiator.digest.return_value = None + assert not cycle.post_digest(radiator, _Slack(), mock.Mock()) + + +def test_the_board_and_digest_go_to_the_radiators_own_channel(monkeypatch): + monkeypatch.setenv("SLACK_WEBHOOK_URL", "https://hooks.slack.com/old") + monkeypatch.setenv("SLACK_TOKEN", "xoxb") + monkeypatch.setenv("SLACK_CHANNEL", "C0OLD") + monkeypatch.delenv("RADIATOR_SLACK_TOKEN", raising=False) + monkeypatch.delenv("RADIATOR_SLACK_WEBHOOK_URL", raising=False) + monkeypatch.delenv("RADIATOR_SLACK_CHANNEL", raising=False) + monkeypatch.delenv("ZE_BASE_URL", raising=False) + monkeypatch.delenv("ZE_REFRESH_TOKEN", raising=False) + assert cycle._slacker("out", mock.Mock()).channel == "C0OLD" + # while SLACK_CHANNEL is Zebrunner's, the radiator's posts don't go there + monkeypatch.setenv("ZE_BASE_URL", "https://zebrunner.example.org") + monkeypatch.setenv("ZE_REFRESH_TOKEN", "tok") + assert cycle._slacker("out", mock.Mock()) is None + monkeypatch.setenv("RADIATOR_SLACK_WEBHOOK_URL", "https://hooks.slack.com/new") + monkeypatch.setenv("RADIATOR_SLACK_CHANNEL", "C0NEW") + assert cycle._slacker("out", mock.Mock()).channel == "C0NEW" + + +def test_the_board_comes_from_the_radiator_the_steps_reported_to(monkeypatch): + monkeypatch.delenv("RADIATOR_URL", raising=False) + monkeypatch.delenv("RADIATOR_TOKEN", raising=False) + with mock.patch("test_harness.cycle.subprocess.run") as run, mock.patch( + "test_harness.cycle._slacker", return_value=mock.Mock() + ), mock.patch("test_harness.cycle.post_cycle_report") as post: + run.return_value = mock.Mock(returncode=0) + cli( + [ + "--acceptance", + "a", + "--", + "--radiator_url=http://r", + "--radiator_token", + "t", + ] + ) + radiator = post.call_args.args[2] + assert (radiator.base_url, radiator.token) == ("http://r", "t") + + +def test_a_slack_failure_after_the_cycle_is_a_warning(monkeypatch, caplog): + monkeypatch.setenv("RADIATOR_URL", "http://radiator") + monkeypatch.setenv("RADIATOR_TOKEN", "tok") + with mock.patch("test_harness.cycle.subprocess.run") as run, mock.patch( + "test_harness.cycle._slacker", return_value=mock.Mock() + ), mock.patch( + "test_harness.cycle.post_cycle_report", side_effect=RuntimeError("slack down") + ): + run.return_value = mock.Mock(returncode=0) + assert cli(["--acceptance", "a"]) == 0 + assert "Couldn't post the cycle board: RuntimeError: slack down" in caplog.text diff --git a/tests/test_local.py b/tests/test_local.py index 2f5eeb8..31cf6b4 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -144,10 +144,12 @@ def fake_run_tests(tests, reporter, collector, logger, args): ) saved = os.listdir(tmp_path) - # acceptance json + csv and the json report were all saved with the prefix - assert len(saved) == 3 + # acceptance json + csv, the json report, and the run saved for the + # Information Radiator were all saved with the prefix + assert len(saved) == 4 assert all(name.startswith("aragorn_") for name in saved) assert "aragorn_test_report.json" in saved + assert any(name.startswith("aragorn_radiator_") for name in saved) def test_target_override_requires_both_flags(mocker, tmp_path): diff --git a/tests/test_query_parameters.py b/tests/test_query_parameters.py new file mode 100644 index 0000000..5fe4ec9 --- /dev/null +++ b/tests/test_query_parameters.py @@ -0,0 +1,180 @@ +"""Query parameters: given once, sent with every query.""" + +import json +import os + +import pytest +from pytest_httpx import HTTPXMock +from translator_testing_model.datamodel.pydanticmodel import ( + PathfinderTestAsset, + TestAsset, +) + +from radiator_schema import RunPayload +from test_harness.main import _query_parameters, main +from test_harness.run import run_tests +from test_harness.runner.generate_query import generate_query +from test_harness.trapi import parse_query_parameters, validate_query_parameters + +from .helpers.example_tests import example_test_cases +from .helpers.logger import setup_logger +from .helpers.mock_responses import kp_response +from .helpers.mocks import MockReporter, MockResultCollector + +logger = setup_logger() + +ONE_HOP = TestAsset.model_validate( + { + "id": "Asset_1", + "input_id": "MONDO:0004979", + "input_category": "biolink:Disease", + "predicate_id": "biolink:treats", + "output_id": "CHEBI:31690", + "expected_output": "TopAnswer", + "qualifiers": [], + } +) + + +def test_parse_json_and_files(tmp_path): + assert parse_query_parameters('{"timeout": 300}') == {"timeout": 300} + assert parse_query_parameters("") is None + assert parse_query_parameters(None) is None + path = tmp_path / "params.json" + path.write_text('{"bypass_cache": true}') + assert parse_query_parameters(f"@{path}") == {"bypass_cache": True} + with pytest.raises(ValueError, match="valid JSON"): + parse_query_parameters("{timeout: 300}") + with pytest.raises(ValueError, match="JSON object"): + parse_query_parameters("[1, 2]") + + +def test_validation_trapi_2(): + params = {"timeout": 300, "log_level": "DEBUG", "my_service_flag": "x"} + # the standard ones are checked; a service's own ones pass through + assert validate_query_parameters(params, "2.0.0") == { + "timeout": 300.0, + "log_level": "DEBUG", + "my_service_flag": "x", + } + with pytest.raises(ValueError, match="timeout"): + validate_query_parameters({"timeout": "soon"}, "2.0.0") + with pytest.raises(ValueError, match="log_level"): + validate_query_parameters({"log_level": "LOUD"}, "2.0.0") + + +def test_validation_trapi_1_6(): + assert validate_query_parameters({"bypass_cache": True}, "1.6.0") == { + "bypass_cache": True + } + with pytest.raises(ValueError, match="can't carry timeout"): + validate_query_parameters({"timeout": 30}, "1.6.0") + + +def test_queries_carry_the_parameters(): + params = {"timeout": 300.0, "bypass_cache": True} + query = generate_query(ONE_HOP, "2.0.0", params) + assert query["parameters"] == params + assert "timeout" not in query + + legacy = generate_query(ONE_HOP, "1.6.0", {"bypass_cache": True}) + assert legacy["bypass_cache"] is True + assert "parameters" not in legacy + + pathfinder = PathfinderTestAsset.model_validate( + { + "id": "Asset_2", + "source_input_id": "MONDO:1", + "target_input_id": "CHEBI:2", + "predicate_id": "biolink:related_to", + "expected_output": "TopAnswer", + "path_nodes": [], + "minimum_required_path_nodes": 1, + } + ) + assert generate_query(pathfinder, "2.0.0", params)["parameters"] == params + # none given: the query is unchanged + assert "parameters" not in generate_query(ONE_HOP, "2.0.0") + + +def test_every_query_sent_carries_them(httpx_mock: HTTPXMock): + httpx_mock.add_response(url="http://localhost:8080/query", json=kp_response) + httpx_mock.add_response( + url="https://nodenorm-es.ci.transltr.io/get_normalized_nodes", + # nothing normalizes, so every curie is kept as is + json={ + curie: None + for curie in [ + "MONDO:0010794", + "DRUGBANK:DB00313", + "MESH:D001463", + "CHEBI:18295", + "CHEBI:31690", + "CL:0000097", + "MONDO:0004979", + "NCBIGene:3815", + "NCBIGene:4254", + "PR:000049994", + ] + }, + ) + run_tests( + tests=example_test_cases, + reporter=MockReporter(base_url="http://test"), + collector=MockResultCollector("ci", logger, target="aragorn"), + logger=logger, + args={ + "suite": "testing", + "trapi_version": "2.0.0", + "target_url": "http://localhost:8080", + "target": "aragorn", + "query_parameters": {"timeout": 300.0}, + }, + ) + sent = [ + json.loads(r.content) + for r in httpx_mock.get_requests() + if r.url.path == "/query" + ] + assert sent and all(q["parameters"] == {"timeout": 300.0} for q in sent) + + +def _main_args(tmp_path, **extra): + return { + "tests": example_test_cases, + "suite": "testing", + "save_to_dashboard": False, + "json_output": False, + "log_level": "ERROR", + "local": True, + "output_dir": str(tmp_path), + **extra, + } + + +def test_main_refuses_bad_parameters_before_running(mocker, tmp_path): + run = mocker.patch("test_harness.main.run_tests") + code = main( + _main_args(tmp_path, trapi_version="1.6.0", query_parameters={"timeout": 30}) + ) + assert code == 1 + run.assert_not_called() + + +def test_main_sends_and_records_them(mocker, tmp_path): + """The run's parameters reach the queries and the radiator's record.""" + run = mocker.patch("test_harness.main.run_tests") + assert main(_main_args(tmp_path, query_parameters={"timeout": 300})) == 0 + assert run.call_args.args[4]["query_parameters"] == {"timeout": 300.0} + [saved] = [p for p in os.listdir(tmp_path) if p.startswith("radiator_")] + with open(tmp_path / saved) as f: + payload = RunPayload.model_validate(json.load(f)) + assert payload.run.query_parameters == {"timeout": 300.0} + + +def test_cli_flag_and_environment(monkeypatch): + assert _query_parameters('{"timeout": 5}') == {"timeout": 5} + import argparse + + with pytest.raises(argparse.ArgumentTypeError): + _query_parameters("not json") diff --git a/tests/test_radiator.py b/tests/test_radiator.py new file mode 100644 index 0000000..88fa4b9 --- /dev/null +++ b/tests/test_radiator.py @@ -0,0 +1,900 @@ +"""Tests for reporting to the new Information Radiator. + +Covers the structured records the collector builds (including the response +metadata Zebrunner never got: n_results, HTTP status, response time), and the +client's batching, auth, failure handling, and save/replay. +""" + +import json + +import httpx +import os +from uuid import uuid4 + +import pytest +from pytest_httpx import HTTPXMock + +from radiator_schema import AssetResult, RunCreate, RunPayload +from test_harness.main import main +from test_harness.radiator_client import RadiatorClient, push +from test_harness.result_collector import ResultCollector +from test_harness.run import record_response_meta, run_tests +from test_harness.utils import AgentReport, AgentStatus, TestReport + +from .helpers.example_tests import example_test_cases +from .helpers.logger import setup_logger +from .helpers.mock_responses import kp_response +from .helpers.mocks import MockReporter, MockResultCollector, MockSlacker + +logger = setup_logger() + +BASE = "http://radiator" +INGEST = f"{BASE}/api/ingest" + + +class _Asset: + name = "asset-name" + description = None + id = "asset-1" + expected_output = "TopAnswer" + predicate_id = "biolink:treats" + input_id = "MONDO:1" + output_id = "CHEBI:2" + + +class _Case: + id = "case-1" + + +def _run(**kwargs): + return RunCreate( + run_id=uuid4(), suite="suite", started_at="2026-10-06T00:00:00Z", **kwargs + ) + + +def _recording_client(): + """A client that records but sends nothing, with a run open.""" + client = RadiatorClient(enabled=False) + client.start_run(_run()) + return client + + +def _report(status, **kwargs): + return AgentReport(status=status, message=None, actual_output=None, **kwargs) + + +def test_record_response_meta_counts_results(): + report = _report(AgentStatus.SKIPPED) + record_response_meta( + report, + { + "response": {"message": {"results": [{}, {}, {}]}}, + "status_code": 200, + "elapsed_s": 1.5, + }, + ) + assert report.http_status == 200 + assert report.n_results == 3 + assert report.response_time_s == 1.5 + + +def test_record_response_meta_ignores_stand_in_results_on_error(): + """An errored agent gets an empty stand-in result list from the query + runner; that must not read as the agent answering with zero results.""" + report = _report(AgentStatus.SKIPPED) + record_response_meta( + report, {"response": {"message": {"results": []}}, "status_code": 598} + ) + assert report.http_status == 598 + assert report.n_results is None + assert report.response_time_s is None + + +def test_collector_builds_structured_asset_result(): + client = _recording_client() + collector = ResultCollector("dev", logger, radiator=client) + report = TestReport( + pks={"parent_pk": "parent", "ars": "merged", "shepherd-arax": "arax-pk"}, + result={ + "ars": _report(AgentStatus.PASSED, http_status=200, n_results=120), + "shepherd-arax": _report( + AgentStatus.NO_RESULTS, http_status=200, n_results=0 + ), + }, + test_details=None, + ) + report.result["ars"].actual_output = { + "found": True, + "ars_rank": 4, + "ars_score": 0.91, + } + collector.collect_acceptance_result(_Case(), _Asset(), report, "parent", "url") + + [result] = client.payload.results + assert result.test_case_id == "case-1" + assert result.asset_id == "asset-1" + assert result.kind == "acceptance" + assert result.input_curie == "MONDO:1" + assert result.output_curie == "CHEBI:2" + # derived from the ARS, as run_tests does + assert result.status == "PASSED" + assert result.parent_pk == "parent" + + agents = {agent.agent: agent for agent in result.agents} + # one row per agent on the roster, like the CSV + assert list(agents) == collector.agents + ars = agents["ars"] + assert (ars.found, ars.rank, ars.score) == (True, 4, 0.91) + assert ars.n_results == 120 + assert ars.pk == "merged" + arax = agents["shepherd-arax"] + assert arax.status == "NO_RESULTS" + assert arax.found is False + assert arax.n_results == 0 + # no response at all: skipped, and nothing is claimed about it + bte = agents["shepherd-bte"] + assert bte.status == "SKIPPED" + assert bte.found is None and bte.n_results is None + + +def test_collector_survives_a_record_the_schema_rejects(): + """A malformed value costs the radiator that asset, not the run.""" + client = _recording_client() + collector = ResultCollector("dev", logger, radiator=client) + report = TestReport( + pks={}, + result={"ars": _report(AgentStatus.PASSED)}, + test_details=None, + ) + report.result["ars"].actual_output = {"found": True, "ars_rank": "first"} + collector.collect_acceptance_result(_Case(), _Asset(), report, None, "url") + + assert client.payload.results == [] + # the existing outputs are unaffected + assert collector.acceptance_stats["ars"]["TopAnswer"]["PASSED"] == 1 + + +def test_collector_force_skipped_skips_every_agent(): + client = _recording_client() + collector = ResultCollector("dev", logger, radiator=client) + report = TestReport( + pks={"ars": "None"}, + result={"ars": _report(AgentStatus.FAILED, http_status=500)}, + test_details=None, + ) + collector.collect_acceptance_result( + _Case(), _Asset(), report, None, "url", force_skipped=True + ) + + [result] = client.payload.results + assert result.status == "SKIPPED" + assert {agent.status for agent in result.agents} == {"SKIPPED"} + # the query runner's "None" placeholder is not a pk + assert all(agent.pk is None for agent in result.agents) + + +def test_collector_records_failed_performance_checkpoint(): + client = _recording_client() + collector = ResultCollector("ci", logger, radiator=client) + collector.collect_performance_result( + _Case(), + _Asset(), + "url", + "https://ars.ci.transltr.io", + { + "helmsdeep_target": "ars", + "exit_code": 0, + "summary": { + "checkpoints_passed": False, + "max_sustainable_concurrency": 12.5, + }, + }, + ) + + [perf] = client.payload.performance + assert perf.status == "FAILED" + assert perf.checkpoints_passed is False + assert perf.max_sustainable_concurrency == 12.5 + assert perf.summary["checkpoints_passed"] is False + + +def test_run_tests_records_response_metadata(httpx_mock: HTTPXMock): + """End to end through run_tests: a direct query's result count and + response time reach the radiator record.""" + httpx_mock.add_response(url="http://localhost:8080/query", json=kp_response) + httpx_mock.add_response( + url="https://nodenorm-es.ci.transltr.io/get_normalized_nodes", + # nothing normalizes, so every curie is kept as is + json={ + curie: None + for curie in [ + "MONDO:0010794", + "DRUGBANK:DB00313", + "MESH:D001463", + "CHEBI:18295", + "CHEBI:31690", + "CL:0000097", + "MONDO:0004979", + "NCBIGene:3815", + "NCBIGene:4254", + "PR:000049994", + ] + }, + ) + client = _recording_client() + collector = MockResultCollector("ci", logger, target="aragorn", radiator=client) + run_tests( + tests=example_test_cases, + reporter=MockReporter(base_url="http://test"), + collector=collector, + logger=logger, + args={ + "suite": "testing", + "trapi_version": "1.6.0", + "target_url": "http://localhost:8080", + "target": "aragorn", + }, + ) + + assert client.payload.results + for result in client.payload.results: + [aragorn] = result.agents + assert aragorn.agent == "aragorn" + assert aragorn.http_status == 200 + assert aragorn.n_results == len(kp_response["message"]["results"]) + assert aragorn.response_time_s is not None + + +def test_links_use_the_public_url(monkeypatch): + """Uploads go to RADIATOR_URL; links people click use RADIATOR_PUBLIC_URL.""" + monkeypatch.setenv("RADIATOR_PUBLIC_URL", "http://localhost:8000/") + client = RadiatorClient(base_url="http://radiator:8000", token="tok") + run = _run() + client.payload = RunPayload(run=run) + assert client.base_url == "http://radiator:8000" + assert client.run_url == f"http://localhost:8000/runs/{run.run_id}" + assert client.diff_url == f"http://localhost:8000/runs/{run.run_id}/diff" + + +def test_client_needs_url_and_token(): + assert not RadiatorClient.is_configured() + assert not RadiatorClient.is_configured(base_url=BASE) + assert RadiatorClient.is_configured(base_url=BASE, token="tok") + assert not RadiatorClient(base_url=BASE).enabled + assert not RadiatorClient(base_url=BASE, token="tok", enabled=False).enabled + + +def test_disabled_client_sends_nothing(httpx_mock: HTTPXMock): + client = RadiatorClient(base_url=BASE, token="tok", enabled=False) + client.start_run(_run()) + client.add_result(_asset_result()) + client.finish_run() + assert httpx_mock.get_requests() == [] + assert client.run_url is None + assert len(client.payload.results) == 1 + + +def _asset_result(asset_id="asset-1"): + return AssetResult( + test_case_id="case-1", asset_id=asset_id, kind="acceptance", status="PASSED" + ) + + +def test_client_batches_results(httpx_mock: HTTPXMock): + run = _run() + httpx_mock.add_response(method="POST", url=f"{INGEST}/runs") + httpx_mock.add_response(method="POST", url=f"{INGEST}/runs/{run.run_id}/results") + httpx_mock.add_response(method="POST", url=f"{INGEST}/runs/{run.run_id}/results") + httpx_mock.add_response(method="PATCH", url=f"{INGEST}/runs/{run.run_id}") + + client = RadiatorClient(base_url=BASE + "/", token="tok", batch_size=2) + client.start_run(run) + for i in range(3): + client.add_result(_asset_result(f"asset-{i}")) + client.finish_run(counts={"PASSED": 3}) + + requests = httpx_mock.get_requests() + assert [(r.method, r.url.path) for r in requests] == [ + ("POST", "/api/ingest/runs"), + ("POST", f"/api/ingest/runs/{run.run_id}/results"), + ("POST", f"/api/ingest/runs/{run.run_id}/results"), + ("PATCH", f"/api/ingest/runs/{run.run_id}"), + ] + assert all(r.headers["Authorization"] == "Bearer tok" for r in requests) + # a full batch of two, then the remainder on finish + batches = [json.loads(r.content)["results"] for r in requests[1:3]] + assert [len(batch) for batch in batches] == [2, 1] + assert json.loads(requests[3].content)["counts"] == {"PASSED": 3} + assert not client.upload_failed + assert client.run_url == f"{BASE}/runs/{run.run_id}" + + +def test_client_failure_stops_uploads_and_replays(httpx_mock: HTTPXMock, tmp_path): + """A failed upload never raises, stops further uploads, and the saved run + replays in full, keeping its original end time.""" + run = _run() + httpx_mock.add_response(method="POST", url=f"{INGEST}/runs", status_code=503) + + client = RadiatorClient(base_url=BASE, token="tok", batch_size=1) + client.start_run(run) + client.add_result(_asset_result("asset-1")) + client.add_result(_asset_result("asset-2")) + client.finish_run(counts={"PASSED": 2}) + + assert client.upload_failed + assert len(httpx_mock.get_requests()) == 1 + path = client.save(str(tmp_path), prefix="aragorn_") + assert path.endswith(f"aragorn_radiator_{run.run_id}.json") + + with open(path) as f: + payload = RunPayload.model_validate(json.load(f)) + assert len(payload.results) == 2 + + httpx_mock.reset(assert_all_responses_were_requested=False) + httpx_mock.add_response(method="POST", url=f"{INGEST}/runs") + httpx_mock.add_response(method="POST", url=f"{INGEST}/runs/{run.run_id}/results") + httpx_mock.add_response(method="PATCH", url=f"{INGEST}/runs/{run.run_id}") + replay = RadiatorClient(base_url=BASE, token="tok") + assert push(payload, replay) + + requests = httpx_mock.get_requests() + assert len(json.loads(requests[1].content)["results"]) == 2 + finish = json.loads(requests[2].content) + assert finish["ended_at"] == payload.finish.model_dump(mode="json")["ended_at"] + assert finish["counts"] == {"PASSED": 2} + + +def test_record_response_meta_ignores_a_non_numeric_status(): + """After a failed ARS poll the status is the ARS's own string; that must + not invalidate the asset's whole radiator record.""" + report = _report(AgentStatus.SKIPPED) + record_response_meta( + report, {"response": {"message": {"results": []}}, "status_code": "Running"} + ) + assert report.http_status is None + assert report.n_results is None + + +def test_collector_records_a_performance_run_that_raised(): + client = _recording_client() + collector = ResultCollector("ci", logger, radiator=client) + collector.record_performance_error(_Case(), _Asset(), "https://ars", "boom") + [perf] = client.payload.performance + assert (perf.status, perf.error, perf.host) == ("FAILED", "boom", "https://ars") + + +class _BrokenZebrunner(MockReporter): + def create_test(self, test, asset): + raise RuntimeError("Zebrunner is down") + + +def test_run_tests_still_collects_when_zebrunner_fails(httpx_mock: HTTPXMock): + """A Zebrunner failure for an asset must not drop it from the radiator.""" + httpx_mock.add_response(url="http://localhost:8080/query", json=kp_response) + httpx_mock.add_response( + url="https://nodenorm-es.ci.transltr.io/get_normalized_nodes", + json={ + curie: None + for curie in [ + "MONDO:0010794", + "DRUGBANK:DB00313", + "MESH:D001463", + "CHEBI:18295", + "CHEBI:31690", + "CL:0000097", + "MONDO:0004979", + "NCBIGene:3815", + "NCBIGene:4254", + "PR:000049994", + ] + }, + ) + client = _recording_client() + collector = MockResultCollector("ci", logger, target="aragorn", radiator=client) + run_tests( + tests=example_test_cases, + reporter=_BrokenZebrunner(base_url="http://test"), + collector=collector, + logger=logger, + args={ + "suite": "testing", + "trapi_version": "1.6.0", + "target_url": "http://localhost:8080", + "target": "aragorn", + }, + ) + assert client.payload.results + + +def _main_args(tmp_path): + return { + "tests": example_test_cases, + "suite": "testing", + "save_to_dashboard": False, + "json_output": False, + "log_level": "ERROR", + "output_dir": str(tmp_path), + } + + +def _saved_payload(tmp_path): + [path] = [p for p in os.listdir(tmp_path) if p.startswith("radiator_")] + with open(os.path.join(tmp_path, path)) as f: + return RunPayload.model_validate(json.load(f)) + + +def test_main_survives_zebrunner_being_down(mocker, monkeypatch, tmp_path): + """Zebrunner refusing to open a run no longer stops the run.""" + monkeypatch.setenv("ZE_BASE_URL", "http://zebrunner") + monkeypatch.setenv("ZE_REFRESH_TOKEN", "tok") + mocker.patch( + "test_harness.main.Reporter.get_auth", side_effect=RuntimeError("down") + ) + run = mocker.patch("test_harness.main.run_tests") + assert main(_main_args(tmp_path)) == 0 + run.assert_called_once() + assert _saved_payload(tmp_path).finish is not None + + +def test_main_saves_the_radiator_run_before_slack_can_fail(mocker, tmp_path): + """The radiator run is closed and saved before Zebrunner and Slack are + finished with, so their failures can't cost it the run.""" + mocker.patch("test_harness.main.run_tests") + mocker.patch( + "test_harness.main.LocalReporter.finish_test_run", + side_effect=RuntimeError("Zebrunner said no"), + ) + with pytest.raises(RuntimeError): + main(_main_args(tmp_path)) + assert _saved_payload(tmp_path).finish is not None + + +def test_main_keeps_a_crashed_run_open(mocker, tmp_path): + """A run that died partway is saved, but not marked finished.""" + mocker.patch("test_harness.main.run_tests", side_effect=RuntimeError("crash")) + with pytest.raises(RuntimeError): + main(_main_args(tmp_path)) + assert _saved_payload(tmp_path).finish is None + + +class _RecordingSlacker(MockSlacker): + def __init__(self): + self.messages = [] + self.files = [] + self.results = [] + + def post_notification(self, messages=[]): + self.messages.extend(messages) + + def upload_binary_file(self, filename, content, initial_comment=None, title=None): + self.files.append((filename, content, initial_comment)) + + def upload_test_results_file(self, filename, extension, results): + self.results.append(extension) + + +def _main_with_slack( + mocker, + monkeypatch, + tmp_path, + *, + radiator=True, + summary=None, + acceptance=True, + performance=False, + args=None, + new_slacker=None, + zebrunner=False, +): + def fake_run_tests(tests, reporter, collector, logger, args): + collector.has_acceptance_results = acceptance + collector.has_performance_results = performance + + mocker.patch("test_harness.main.run_tests", side_effect=fake_run_tests) + slacker = _RecordingSlacker() + mocker.patch("test_harness.main.Slacker", return_value=slacker) + mocker.patch("test_harness.main.Slacker.is_configured", return_value=True) + mocker.patch("test_harness.main.radiator_slacker", return_value=new_slacker) + if zebrunner: + # configured and reachable: a run with a Zebrunner link + reporter = mocker.patch("test_harness.main.Reporter").return_value + mocker.patch("test_harness.main.Reporter.is_configured", return_value=True) + reporter.is_local = False + reporter.base_path = "https://zebrunner.example.org" + reporter.test_run_id = 42 + reporter.test_name = "zebrunner_run" + if radiator: + monkeypatch.setenv("RADIATOR_URL", "http://radiator") + monkeypatch.setenv("RADIATOR_TOKEN", "tok") + mocker.patch("test_harness.main.RadiatorClient._send") + mocker.patch("test_harness.main.RadiatorClient.summary", return_value=summary) + mocker.patch( + "test_harness.main.RadiatorClient.history_png", return_value=b"\x89PNG" + ) + mocker.patch( + "test_harness.main.RadiatorClient.performance_png", + return_value=b"\x89PNG perf", + ) + assert main({**_main_args(tmp_path), **(args or {})}) == 0 + return slacker + + +def test_slack_report_carries_the_radiator_history(mocker, monkeypatch, tmp_path): + slacker = _main_with_slack( + mocker, + monkeypatch, + tmp_path, + summary={ + "pass_rate": 0.86, + "previous_pass_rate": 0.9, + "regressions": 3, + "fixed": 1, + }, + ) + report = slacker.messages[-1] + assert "Pass rate 86% (-4.0 pts vs the previous run)" in report + assert "3 regressions · 1 fixed" in report + assert "/diff|What changed>" in report + [(filename, content, comment)] = slacker.files + assert filename == "pass_rate_history.png" and content == b"\x89PNG" + assert "Open in the Information Radiator" in comment + + +def test_runs_of_a_cycle_carry_its_id(mocker, monkeypatch, tmp_path): + from uuid import UUID + + cycle_id = UUID("1b4e28ba-2fa1-11d2-883f-0016d3cca427") + _main_with_slack( + mocker, monkeypatch, tmp_path, radiator=False, args={"cycle_id": cycle_id} + ) + run = _saved_payload(tmp_path).run + assert run.cycle_id == cycle_id + # for the dashboard's progress bar while the run is going + assert run.expected_assets == sum( + len(test.test_assets) for test in example_test_cases.values() + ) + + +def test_slack_report_first_run_of_a_series(mocker, monkeypatch, tmp_path): + slacker = _main_with_slack( + mocker, + monkeypatch, + tmp_path, + summary={"pass_rate": 0.5, "previous_pass_rate": None}, + ) + assert "Pass rate 50%\n" in slacker.messages[-1] + assert "What changed" not in slacker.messages[-1] + + +def test_slack_report_without_the_radiator(mocker, monkeypatch, tmp_path): + """No radiator: the report is what it always was.""" + slacker = _main_with_slack(mocker, monkeypatch, tmp_path, radiator=False) + assert "Pass rate" not in slacker.messages[-1] + assert slacker.files == [] + + +def test_slack_report_carries_each_services_concurrency(mocker, monkeypatch, tmp_path): + slacker = _main_with_slack( + mocker, + monkeypatch, + tmp_path, + acceptance=False, + performance=True, + summary={ + "pass_rate": None, + "performance": [ + { + "host": "https://arax.ci.transltr.io", + "helmsdeep_target": "arax", + "profile": "mixed", + "max_sustainable_concurrency": 7.5, + "previous_max_sustainable_concurrency": 7.2, + "checkpoints_passed": False, + } + ], + }, + ) + report = slacker.messages[-1] + assert ( + "arax (mixed): max sustainable concurrency 7.5 (+0.3 vs the previous run)" + " · checkpoints missed" in report + ) + assert "Pass rate" not in report + [(filename, content, comment)] = slacker.files + assert filename == "concurrency_history.png" and content == b"\x89PNG perf" + + +def test_performance_line_variants(): + from test_harness.main import _performance_line + + base = {"host": "https://ars", "helmsdeep_target": "ars", "profile": None} + assert _performance_line({**base, "error": "boom"}) == "ars: run failed: boom" + assert ( + _performance_line({**base, "max_sustainable_concurrency": None}) + == "ars: no stage met the SLO" + ) + first = _performance_line( + {**base, "max_sustainable_concurrency": 12.0, "knee_unsupported": True} + ) + assert ( + first + == "ars: max sustainable concurrency 12.0 (unsupported, see stage warnings)" + ) + + +def test_slack_links_only_to_where_the_run_went(mocker, monkeypatch, tmp_path): + """Without Zebrunner, the report links to the new radiator only.""" + slacker = _main_with_slack( + mocker, monkeypatch, tmp_path, summary={"pass_rate": 1.0} + ) + for message in slacker.messages: + assert "/test-runs/local" not in message + assert "View in the new Information Radiator" in message + + +def test_results_reach_the_radiator_as_each_test_case_finishes(httpx_mock: HTTPXMock): + """A real run can be fewer results than one batch while taking an hour; + each test case's results are sent as soon as it finishes.""" + httpx_mock.add_response(url="http://localhost:8080/query", json=kp_response) + httpx_mock.add_response( + url="https://nodenorm-es.ci.transltr.io/get_normalized_nodes", + json={ + curie: None + for curie in [ + "MONDO:0010794", + "DRUGBANK:DB00313", + "MESH:D001463", + "CHEBI:18295", + "CHEBI:31690", + "CL:0000097", + "MONDO:0004979", + "NCBIGene:3815", + "NCBIGene:4254", + "PR:000049994", + ] + }, + ) + client = _recording_client() + sent = [] + client.flush = lambda: ( + sent.append(len(client._pending.results)), + client._pending.results.clear(), + ) + collector = MockResultCollector("ci", logger, target="aragorn", radiator=client) + run_tests( + tests=example_test_cases, + reporter=MockReporter(base_url="http://test"), + collector=collector, + logger=logger, + args={ + "suite": "testing", + "trapi_version": "1.6.0", + "target_url": "http://localhost:8080", + "target": "aragorn", + }, + ) + # one flush per test case, each carrying that case's assets + assert len(sent) == len(example_test_cases) + assert sum(sent) == len(client.payload.results) > 0 + + +def test_the_radiator_can_have_its_own_slack_channel(mocker, monkeypatch, tmp_path): + """Alongside Zebrunner: the old channel keeps the report it always got, and + the radiator's report, with its headline and charts, goes to its own.""" + new = _RecordingSlacker() + old = _main_with_slack( + mocker, + monkeypatch, + tmp_path, + summary={ + "pass_rate": 0.86, + "previous_pass_rate": 0.9, + "regressions": 3, + "fixed": 1, + }, + new_slacker=new, + zebrunner=True, + ) + for message in old.messages: + assert "new Information Radiator" not in message + assert "Pass rate" not in message + assert old.files == [] # no radiator charts + # both channels get the result files + assert old.results == new.results == ["json", "csv"] + + assert all("View in the new Information Radiator" in m for m in new.messages) + assert "Pass rate 86% (-4.0 pts vs the previous run)" in new.messages[-1] + assert [f for f, _, _ in new.files] == ["pass_rate_history.png"] + + +def test_a_failing_radiator_channel_never_costs_the_old_one( + mocker, monkeypatch, tmp_path +): + class Broken(_RecordingSlacker): + def post_notification(self, messages=[]): + raise RuntimeError("channel_not_found") + + old = _main_with_slack( + mocker, + monkeypatch, + tmp_path, + summary={"pass_rate": 1.0}, + new_slacker=Broken(), + zebrunner=True, + ) + assert len(old.messages) == 2 # started, and the report + + +def test_radiator_slack_settings(monkeypatch, caplog): + from test_harness.slacker import radiator_slacker + + for var in ( + "RADIATOR_SLACK_WEBHOOK_URL", + "RADIATOR_SLACK_CHANNEL", + "RADIATOR_SLACK_TOKEN", + ): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("SLACK_TOKEN", "xoxb-shared") + assert radiator_slacker() is None + + monkeypatch.setenv("RADIATOR_SLACK_WEBHOOK_URL", "https://hooks.slack.com/x") + assert radiator_slacker() is None + assert "only partly set" in caplog.text + + monkeypatch.setenv("RADIATOR_SLACK_CHANNEL", "C0NEW") + slacker = radiator_slacker() + assert (slacker.url, slacker.channel) == ("https://hooks.slack.com/x", "C0NEW") + assert slacker.client.token == "xoxb-shared" # the same bot by default + monkeypatch.setenv("RADIATOR_SLACK_TOKEN", "xoxb-other") + assert radiator_slacker().client.token == "xoxb-other" + + +def test_alongside_zebrunner_the_old_channel_keeps_its_report( + mocker, monkeypatch, tmp_path +): + """No channel of the radiator's own: SLACK_CHANNEL is still Zebrunner's, + and gets nothing of the new radiator's.""" + old = _main_with_slack( + mocker, + monkeypatch, + tmp_path, + summary={ + "pass_rate": 0.86, + "previous_pass_rate": 0.9, + "regressions": 3, + "fixed": 1, + }, + zebrunner=True, + ) + for message in old.messages: + assert ( + "zebrunner.example.org/test-runs/42|View in the Information Radiator" + in message + ) + assert "new Information Radiator" not in message + assert "Pass rate" not in message + assert old.files == [] + assert old.results == ["json", "csv"] + + +def test_without_zebrunner_the_radiator_has_slack_channel( + mocker, monkeypatch, tmp_path +): + """After the cutover (or in docker compose), SLACK_CHANNEL is the radiator's.""" + slacker = _main_with_slack( + mocker, monkeypatch, tmp_path, summary={"pass_rate": 0.5} + ) + assert all("View in the new Information Radiator" in m for m in slacker.messages) + assert "Pass rate 50%" in slacker.messages[-1] + + +# --- what the logs say about the dashboard --------------------------------- + + +def test_a_rejected_upload_logs_the_radiators_reason(httpx_mock: HTTPXMock, caplog): + run = _run() + httpx_mock.add_response(method="POST", url=f"{INGEST}/runs") + httpx_mock.add_response( + method="POST", + url=f"{INGEST}/runs/{run.run_id}/results", + status_code=422, + json={"detail": [{"loc": ["body", "results", 0, "status"], "msg": "bad"}]}, + ) + client = RadiatorClient(base_url=BASE, token="tok", batch_size=1) + client.start_run(run) + client.add_result(_asset_result("asset-1")) + assert client.upload_failed + error = caplog.records[-1].getMessage() + assert "answered 422" in error and '"status"' in error and "bad" in error + assert "1 asset results and 0 performance results" in error + assert str(run.run_id) in error + + +def test_an_unreachable_radiator_logs_the_connection_error( + httpx_mock: HTTPXMock, caplog +): + httpx_mock.add_exception(httpx.ConnectError("Connection refused")) + client = RadiatorClient(base_url=BASE, token="tok") + client.start_run(_run()) + assert "opening the run): ConnectError: Connection refused" in caplog.text + + +def test_the_outcome_names_assets_the_dashboard_is_missing(httpx_mock: HTTPXMock): + run = _run() + httpx_mock.add_response(method="POST", url=f"{INGEST}/runs") + httpx_mock.add_response(method="POST", url=f"{INGEST}/runs/{run.run_id}/results") + client = RadiatorClient(base_url=BASE, token="tok", batch_size=1) + client.start_run(run) + client.add_result(_asset_result("asset-1")) + expected = [("case-1", "asset-1"), ("case-1", "asset-2")] + summary, missing = client.outcome(expected) + assert f"{BASE}/runs/{run.run_id} has 1 of the 2 assets" in summary + assert "sent in 1 batches" in summary + assert "1 assets in the suite have no result" in missing + assert "case-1/asset-2" in missing + + offline = RadiatorClient(enabled=False) + offline.start_run(run) + offline.add_result(_asset_result("asset-1")) + [line, _] = offline.outcome(expected) + assert "not uploaded; 1 of the 2 assets were recorded" in line + + +def test_main_logs_where_the_run_went(mocker, monkeypatch, tmp_path, caplog): + monkeypatch.delenv("RADIATOR_URL", raising=False) + monkeypatch.delenv("RADIATOR_TOKEN", raising=False) + caplog.set_level("INFO") + _main_with_slack( + mocker, monkeypatch, tmp_path, radiator=False, args={"log_level": "INFO"} + ) + assert ( + "Not reporting to the Information Radiator (RADIATOR_URL and " + "RADIATOR_TOKEN aren't set)" in caplog.text + ) + # run_tests is mocked out here, so nothing was recorded: every asset is named + assert "Information Radiator: not uploaded; 0 of the" in caplog.text + assert "have no result for the Information Radiator" in caplog.text + + +def test_auto_trapi_reaches_the_run_and_slack(mocker, monkeypatch, tmp_path): + mocker.patch( + "test_harness.main.registered_trapi_versions", + side_effect=lambda infores, maturity: ["1.6.0"], + ) + slacker = _main_with_slack( + mocker, monkeypatch, tmp_path, radiator=False, args={"trapi_version": "auto"} + ) + assert _saved_payload(tmp_path).run.trapi_version == "1.6.0" + assert "TRAPI 1.6.0)" in slacker.messages[0] + + +def test_slack_headline_flags_a_trapi_change(mocker, monkeypatch, tmp_path): + slacker = _main_with_slack( + mocker, + monkeypatch, + tmp_path, + summary={ + "pass_rate": 0.8, + "previous_pass_rate": 0.9, + "regressions": 4, + "fixed": 0, + "trapi_version": "2.0.0", + "previous_trapi_version": "1.6.0", + }, + ) + assert "TRAPI 1.6 → 2.0 since the previous run" in slacker.messages[-1] + + +def test_radiator_cli_takes_its_flags_after_the_subcommand(mocker, monkeypatch): + from test_harness import radiator_client + + monkeypatch.delenv("RADIATOR_URL", raising=False) + monkeypatch.delenv("RADIATOR_TOKEN", raising=False) + digest = mocker.patch("test_harness.cycle.digest_cli", return_value=0) + with pytest.raises(SystemExit) as exit_: + radiator_client.cli( + ["digest", "--radiator_url", "http://r", "--radiator_token", "t"] + ) + assert exit_.value.code == 0 + client = digest.call_args.args[0] + assert (client.base_url, client.token) == ("http://r", "t") diff --git a/tests/test_reporting.py b/tests/test_reporting.py index 8530552..823356e 100644 --- a/tests/test_reporting.py +++ b/tests/test_reporting.py @@ -550,7 +550,7 @@ def test_performance_test_is_finished_in_radiator(mocker): reporter=reporter, collector=MockResultCollector("ci", logger), logger=logger, - args={"suite": "perf", "trapi_version": "1.6.0"}, + args={"suite": "perf", "trapi_version": "2.0.0"}, ) assert reporter.finished, "performance test was never finished in the radiator" @@ -579,7 +579,7 @@ def test_performance_checkpoint_failure_fails_the_radiator_test(mocker): reporter=reporter, collector=MockResultCollector("ci", logger), logger=logger, - args={"suite": "perf", "trapi_version": "1.6.0"}, + args={"suite": "perf", "trapi_version": "2.0.0"}, ) assert reporter.finished[0][1] == AgentStatus.FAILED.value @@ -604,7 +604,7 @@ def test_performance_run_without_a_summary_fails_the_radiator_test(mocker): reporter=reporter, collector=MockResultCollector("ci", logger), logger=logger, - args={"suite": "perf", "trapi_version": "1.6.0"}, + args={"suite": "perf", "trapi_version": "2.0.0"}, ) assert reporter.finished[0][1] == AgentStatus.FAILED.value @@ -628,7 +628,7 @@ def test_performance_test_finished_failed_on_error(mocker): reporter=reporter, collector=MockResultCollector("ci", logger), logger=logger, - args={"suite": "perf", "trapi_version": "1.6.0"}, + args={"suite": "perf", "trapi_version": "2.0.0"}, ) assert reporter.finished @@ -660,7 +660,7 @@ def test_skipped_test_case_marks_all_assets_and_agents_skipped(mocker): reporter=reporter, collector=collector, logger=logger, - args={"suite": "acceptance", "trapi_version": "1.6.0"}, + args={"suite": "acceptance", "trapi_version": "2.0.0"}, ) # 3 assets total across the two acceptance cases in the fixture. diff --git a/tests/test_run.py b/tests/test_run.py index 3478834..298a402 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -53,7 +53,7 @@ def test_run_tests(mocker, httpx_mock: HTTPXMock): logger=logger, args={ "suite": "testing", - "trapi_version": "1.6.0", + "trapi_version": "2.0.0", }, ) @@ -96,7 +96,7 @@ def test_run_tests_with_target_override(httpx_mock: HTTPXMock): logger=logger, args={ "suite": "testing", - "trapi_version": "1.6.0", + "trapi_version": "2.0.0", "target_url": "http://localhost:8080", "target": "aragorn", }, diff --git a/tests/test_trapi.py b/tests/test_trapi.py new file mode 100644 index 0000000..76d5db6 --- /dev/null +++ b/tests/test_trapi.py @@ -0,0 +1,308 @@ +"""Tests for the TRAPI 2.0 surface: the queries sent and the responses read. + +Queries are checked by validating them against TOM, the TRAPI object model, +for the version they were written in. Responses are checked in both binding +shapes, since the ARS can return either. +""" + +import json + +import pytest +import translator_tom.v1_6 as trapi_1_6 +import translator_tom.v2_0 as trapi_2_0 +from pytest_httpx import HTTPXMock +from translator_testing_model.datamodel.pydanticmodel import ( + PathfinderTestAsset, + TestAsset, +) + +from test_harness.acceptance_test_runner import run_acceptance_pass_fail_analysis +from test_harness.main import _trapi_version +from test_harness.pathfinder_test_runner import pathfinder_pass_fail_analysis +from test_harness.run import run_tests +from test_harness.runner.generate_query import generate_query +from test_harness.runner.query_runner import QueryRunner +from test_harness.trapi import binding_ids, is_trapi_2, trapi_minor_version +from test_harness.utils import AgentReport, AgentStatus, PathfinderReport + +from .helpers.example_tests import example_test_cases +from .helpers.logger import setup_logger +from .helpers.mock_responses import kp_response +from .helpers.mocks import MockReporter, MockResultCollector + +logger = setup_logger() + + +def _asset(predicate, input_id, input_category, qualifiers=(), inferred=True): + return TestAsset.model_validate( + { + "id": "Asset_1", + "test_runner_settings": ["inferred"] if inferred else [], + "input_id": input_id, + "input_category": input_category, + "predicate_id": predicate, + "output_id": "NCBIGene:1", + "qualifiers": [ + {"parameter": parameter, "value": value} + for parameter, value in qualifiers + ], + "expected_output": "TopAnswer", + } + ) + + +MVP2_ASSET = _asset( + "biolink:affects", + "CHEBI:2", + "biolink:ChemicalEntity", + qualifiers=[ + ("biolink_qualified_predicate", "biolink:causes"), + ("biolink_object_aspect_qualifier", "activity"), + ("biolink_object_direction_qualifier", "increased"), + ], +) + +PATHFINDER_ASSET = PathfinderTestAsset.model_validate( + { + "id": "PTFQ_1", + "source_input_id": "CHEBI:31690", + "source_input_category": "biolink:Drug", + "target_input_id": "MONDO:0004979", + "target_input_category": "biolink:Disease", + "predicate_id": "biolink:related_to", + "minimum_required_path_nodes": 1, + "path_nodes": [{"ids": ["NCBIGene:3815"], "name": "KIT"}], + "expected_output": "TopAnswer", + } +) + + +class TestVersions: + def test_minor_version(self): + assert trapi_minor_version("2.0.0") == "2.0" + assert trapi_minor_version("1.6.1") == "1.6" + assert is_trapi_2("2.0.0") + assert not is_trapi_2("1.6.0") + + @pytest.mark.parametrize("version", ["1.5.0", "3.0.0", "latest", ""]) + def test_unsupported_version_is_rejected(self, version): + with pytest.raises(ValueError, match="Unsupported TRAPI version"): + trapi_minor_version(version) + + def test_cli_rejects_unsupported_version(self): + from argparse import ArgumentTypeError + + assert _trapi_version("2.0.0") == "2.0.0" + with pytest.raises(ArgumentTypeError): + _trapi_version("1.5.0") + + def test_query_runner_rejects_unsupported_version(self): + with pytest.raises(ValueError): + QueryRunner(logger, trapi_version="1.5.0") + + +class TestQueries2_0: + """Queries default to TRAPI 2.0.""" + + def test_mvp1(self): + query = generate_query(_asset("biolink:treats", "MONDO:1", "biolink:Disease")) + trapi_2_0.Query.model_validate(query) + qgraph = query["message"]["query_graph"] + assert qgraph["nodes"] == { + "SN": {"categories": ["biolink:ChemicalEntity"]}, + "ON": {"ids": ["MONDO:1"], "categories": ["biolink:Disease"]}, + } + edge = qgraph["edges"]["t_edge"] + assert edge["predicates"] == ["biolink:treats"] + assert edge["knowledge_type"] == "inferred" + assert "constraints" not in edge + + def test_mvp2_qualifiers_are_one_constraint_mapping(self): + query = generate_query(MVP2_ASSET) + trapi_2_0.Query.model_validate(query) + edge = query["message"]["query_graph"]["edges"]["t_edge"] + assert edge["constraints"] == { + "qualifiers": [ + { + "biolink:object_aspect_qualifier": "activity", + "biolink:object_direction_qualifier": "increased", + } + ] + } + # the TRAPI 1.x spelling must not be sent: a 2.0 server rejects it + assert "qualifier_constraints" not in edge + + def test_mvp2_leaves_out_unset_qualifiers(self): + """2.0 forbids an empty qualifier mapping, so none is sent.""" + asset = _asset( + "biolink:affects", + "NCBIGene:1", + "biolink:Gene", + qualifiers=[("biolink_object_direction_qualifier", "decreased")], + inferred=False, + ) + query = generate_query(asset) + trapi_2_0.Query.model_validate(query) + edge = query["message"]["query_graph"]["edges"]["t_edge"] + assert edge["constraints"] == { + "qualifiers": [{"biolink:object_direction_qualifier": "decreased"}] + } + assert "knowledge_type" not in edge + assert query["message"]["query_graph"]["nodes"]["ON"]["ids"] == ["NCBIGene:1"] + + asset.qualifiers = [] + edge = generate_query(asset)["message"]["query_graph"]["edges"]["t_edge"] + assert "constraints" not in edge + + def test_pathfinder(self): + query = generate_query(PATHFINDER_ASSET) + trapi_2_0.Query.model_validate(query) + qgraph = query["message"]["query_graph"] + assert "edges" not in qgraph + assert qgraph["paths"] == {"p0": {"subject": "SN", "object": "ON"}} + assert qgraph["nodes"]["SN"] == { + "ids": ["CHEBI:31690"], + "categories": ["biolink:Drug"], + } + + def test_unsupported_input_category(self): + with pytest.raises(Exception, match="Unsupported input category for MVP1"): + generate_query(_asset("biolink:treats", "CHEBI:1", "biolink:Drug")) + + +class TestQueries1_6: + """A 1.6 query can still be asked for, for services that have not moved.""" + + def test_mvp2(self): + query = generate_query(MVP2_ASSET, "1.6.0") + trapi_1_6.Query.model_validate(query) + edge = query["message"]["query_graph"]["edges"]["t_edge"] + assert edge["qualifier_constraints"] == [ + { + "qualifier_set": [ + { + "qualifier_type_id": "biolink:object_aspect_qualifier", + "qualifier_value": "activity", + }, + { + "qualifier_type_id": "biolink:object_direction_qualifier", + "qualifier_value": "increased", + }, + ] + } + ] + assert "constraints" not in edge + + def test_pathfinder(self): + query = generate_query(PATHFINDER_ASSET, "1.6.0") + trapi_1_6.Query.model_validate(query) + assert query["message"]["query_graph"]["paths"] == { + "p0": {"subject": "SN", "object": "ON"} + } + + +class TestBindings: + def test_binding_ids_reads_both_shapes(self): + assert binding_ids({"ids": ["A", "B"]}) == ["A", "B"] + assert binding_ids([{"id": "A"}, {"id": "B"}]) == ["A", "B"] + assert binding_ids({}) == [] + assert binding_ids([]) == [] + + def _report(self, agent): + return {agent: AgentReport(AgentStatus.SKIPPED, None, None)} + + def test_acceptance_reads_2_0_bindings(self): + results = [ + { + "node_bindings": {"SN": {"ids": ["CHEBI:9"]}, "ON": {"ids": ["X:1"]}}, + "analyses": [{"resource_id": "infores:ara", "score": 0.9}], + }, + { + "node_bindings": { + "SN": {"ids": ["CHEBI:8", "CHEBI:2"]}, + "ON": {"ids": ["X:1"]}, + }, + "analyses": [{"resource_id": "infores:ara", "score": 0.4}], + }, + ] + report = run_acceptance_pass_fail_analysis( + self._report("ara"), "ara", results, "CHEBI:2", "TopAnswer" + ) + assert report["ara"].status == AgentStatus.PASSED + assert report["ara"].actual_output == { + "found": True, + "ars_score": None, + "ars_rank": None, + "ara_score": 0.4, + "ara_rank": 2, + } + + def test_acceptance_never_show_with_2_0_bindings(self): + results = [{"node_bindings": {"SN": {"ids": ["CHEBI:2"]}}, "analyses": []}] + report = run_acceptance_pass_fail_analysis( + self._report("ara"), "ara", results, "CHEBI:2", "NeverShow" + ) + assert report["ara"].status == AgentStatus.FAILED + assert report["ara"].message is None + + @pytest.mark.parametrize( + "path_binding", [{"ids": ["aux0"]}, [{"id": "aux0"}]], ids=["2.0", "1.x"] + ) + def test_pathfinder_reads_path_bindings(self, path_binding): + message = { + "results": [ + { + "node_bindings": {}, + "analyses": [ + { + "resource_id": "infores:ara", + "path_bindings": {"p0": path_binding}, + } + ], + } + ], + "auxiliary_graphs": {"aux0": {"edges": ["e0", "e1"]}}, + "knowledge_graph": { + "edges": { + "e0": {"subject": "CHEBI:31690", "object": "NCBIGene:3815"}, + "e1": {"subject": "NCBIGene:3815", "object": "MONDO:0004979"}, + } + }, + } + report = {"ara": PathfinderReport(AgentStatus.SKIPPED, None, None, "")} + pathfinder_pass_fail_analysis(report, "ara", message, [["NCBIGene:3815"]], 1) + assert report["ara"].status == AgentStatus.PASSED + assert report["ara"].expected_nodes_found == "NCBIGene:3815" + + +def test_run_sends_and_reads_trapi_2_0(httpx_mock: HTTPXMock): + """End to end: a 2.0 query goes out and a 2.0 response is scored.""" + httpx_mock.add_response(url="http://localhost:8080/query", json=kp_response) + httpx_mock.add_response( + url="https://nodenorm-es.ci.transltr.io/get_normalized_nodes", + json={"MONDO:0010794": None, "DRUGBANK:DB00313": None, "MESH:D001463": None}, + ) + collector = MockResultCollector("ci", logger, target="gandalf") + tests = {"TestCase_1": example_test_cases["TestCase_1"]} + run_tests( + tests=tests, + reporter=MockReporter(base_url="http://test"), + collector=collector, + logger=logger, + args={ + "suite": "testing", + "trapi_version": "2.0.0", + "target_url": "http://localhost:8080", + "target": "gandalf", + }, + ) + queries = [ + json.loads(request.content) + for request in httpx_mock.get_requests(url="http://localhost:8080/query") + ] + assert queries + for query in queries: + trapi_2_0.Query.model_validate(query) + # Both assets are NeverShow and kp_response binds neither expected answer, + # so both pass -- reading 2.0 bindings as 1.x lists would fail them instead. + assert collector.acceptance_stats["gandalf"]["NeverShow"]["PASSED"] == 2 diff --git a/tests/test_trapi_auto.py b/tests/test_trapi_auto.py new file mode 100644 index 0000000..2b5cc5a --- /dev/null +++ b/tests/test_trapi_auto.py @@ -0,0 +1,120 @@ +"""--trapi_version auto: each run follows its environment's TRAPI version.""" + +import logging + +import httpx +import pytest + +from test_harness.main import _trapi_change, _trapi_version, resolve_trapi_version +from test_harness.runner.smart_api_registry import registered_trapi_versions + + +def _hit(infores, version, *maturities): + return { + "info": { + "x-translator": {"infores": infores, "component": "ARS"}, + "x-trapi": {"version": version}, + }, + "servers": [ + {"url": f"https://{m}.example", "x-maturity": m} for m in maturities + ], + } + + +HITS = [ + _hit("infores:ars", "1.6.0", "production", "testing"), + _hit("infores:ars", "2.0.0", "development", "staging", "testing"), + _hit("infores:aragorn", "1.6.0", "staging"), +] + + +def test_registered_versions_for_one_service_and_maturity(): + assert registered_trapi_versions("infores:ars", "production", HITS) == ["1.6.0"] + assert registered_trapi_versions("infores:ars", "staging", HITS) == ["2.0.0"] + # mid-rollout, registered twice: newest first + assert registered_trapi_versions("infores:ars", "testing", HITS) == [ + "2.0.0", + "1.6.0", + ] + assert registered_trapi_versions("infores:aragorn", "staging", HITS) == ["1.6.0"] + assert registered_trapi_versions("infores:ars", "nowhere", HITS) == [] + + +@pytest.fixture +def registry(monkeypatch): + def fake(infores, maturity, hits=None): + return registered_trapi_versions(infores, maturity, HITS) + + monkeypatch.setattr("test_harness.main.registered_trapi_versions", fake) + + +@pytest.mark.parametrize( + "env, target, expected", + [ + ("prod", None, "1.6.0"), + ("ci", None, "2.0.0"), + ("test", None, "2.0.0"), # both registered: the newest + ("ci", "aragorn", "1.6.0"), # --target: that service's registration + ("ci", "infores:aragorn", "1.6.0"), + ], +) +def test_auto_follows_the_environment(registry, caplog, env, target, expected): + caplog.set_level(logging.INFO) + args = {"trapi_version": "auto", "target": target} + assert resolve_trapi_version(args, env, logging.getLogger("t")) == expected + assert f"TRAPI {expected}: what" in caplog.text + + +def test_auto_falls_back_to_the_default(registry, caplog): + args = {"trapi_version": "auto", "target": "unknown-service"} + assert resolve_trapi_version(args, "ci", logging.getLogger("t")) == "2.0.0" + assert "isn't registered in SmartAPI for ci (staging)" in caplog.text + + +def test_auto_falls_back_when_smartapi_cant_be_read(monkeypatch, caplog): + def down(*args, **kwargs): + raise httpx.ConnectError("Connection refused") + + monkeypatch.setattr("test_harness.main.registered_trapi_versions", down) + args = {"trapi_version": "auto"} + assert resolve_trapi_version(args, "ci", logging.getLogger("t")) == "2.0.0" + assert "couldn't read the SmartAPI registry" in caplog.text + + +def test_unsupported_registered_versions_are_skipped(monkeypatch): + monkeypatch.setattr( + "test_harness.main.registered_trapi_versions", + lambda infores, maturity: ["1.5.0", "1.6.0"], + ) + args = {"trapi_version": "auto"} + assert resolve_trapi_version(args, "ci", logging.getLogger("t")) == "1.6.0" + + +def test_a_chosen_version_is_used_as_given(monkeypatch): + monkeypatch.setattr( + "test_harness.main.registered_trapi_versions", + lambda *a: pytest.fail("no lookup for an explicit version"), + ) + args = {"trapi_version": "1.6.0"} + assert resolve_trapi_version(args, "ci", logging.getLogger("t")) == "1.6.0" + + +def test_the_cli_accepts_auto(): + assert _trapi_version("auto") == "auto" + assert _trapi_version("1.6.0") == "1.6.0" + + +def test_trapi_change(): + assert _trapi_change("1.6.0", "2.0.0") == ("1.6", "2.0") + assert _trapi_change("2.0.0", "2.0.1") is None + assert _trapi_change(None, "2.0.0") is None + + +def test_auto_falls_back_when_smartapi_answers_garbage(monkeypatch, caplog): + def html(*args, **kwargs): + raise ValueError("Expecting value: line 1 column 1 (char 0)") + + monkeypatch.setattr("test_harness.main.registered_trapi_versions", html) + args = {"trapi_version": "auto"} + assert resolve_trapi_version(args, "ci", logging.getLogger("t")) == "2.0.0" + assert "couldn't read the SmartAPI registry" in caplog.text