From 3216447e52e0b75d996ab1b646aff897f6fbdc67 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 07:28:12 +0000 Subject: [PATCH 1/3] ci(live): file one issue per failing data source from the weekly live run The scheduled live tier was red four weeks running (Sep 29 - Oct 20, 2026). A NASA CMR outage kept it red, so the Yukon API breaking underneath it (29 failed tests on 2026-10-20) went unnoticed. scripts/report_live.py reads the run's JUnit XML, groups the tests by data source (the test file), and files one issue per failing source under the health check's data-source label, with identical errors counted together. A source still failing next week has its issue updated and commented; a source whose tests pass again has it closed; a source whose tests all skipped is left alone. Every run also writes a per-source table to the job summary. Scheduled runs file issues, manual runs only write the summary. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqF35dgJEZwc35XaU8n4iu --- .github/workflows/ci.yml | 19 ++- scripts/report_live.py | 283 +++++++++++++++++++++++++++++++++++ tests/test_live_reporting.py | 138 +++++++++++++++++ 3 files changed, 439 insertions(+), 1 deletion(-) create mode 100644 scripts/report_live.py create mode 100644 tests/test_live_reporting.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 276394d7..9483d690 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,6 +135,9 @@ jobs: if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.run_live) runs-on: ubuntu-latest timeout-minutes: 60 + permissions: + contents: read + issues: write steps: - uses: actions/checkout@v4 with: @@ -154,4 +157,18 @@ jobs: NVE_API_KEY: ${{ secrets.NVE_API_KEY }} AISORI_USERNAME: ${{ secrets.AISORI_USERNAME }} AISORI_PASSWORD: ${{ secrets.AISORI_PASSWORD }} - run: pixi run -e test-py313 test-live -v --tb=short -rfEs + run: pixi run -e test-py313 test-live -v --tb=short -rfEs --junitxml=live-results.xml + + # One red run hides which provider broke (four red weeks in Sep-Oct 2026 + # buried a Yukon API change), so file one issue per failing data source, + # under the health check's `data-source` label, and close it on recovery. + # Scheduled runs file issues; manual runs only write the job summary. + - name: Report failures by data source + if: always() + env: + GITHUB_TOKEN: ${{ github.event_name == 'schedule' && secrets.GITHUB_TOKEN || '' }} + GITHUB_REPOSITORY: ${{ github.repository }} + run: | + pixi run -e test-py313 python scripts/report_live.py live-results.xml \ + --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ + --checked "$(date -u +%Y-%m-%d)" diff --git a/scripts/report_live.py b/scripts/report_live.py new file mode 100644 index 00000000..f1a530a0 --- /dev/null +++ b/scripts/report_live.py @@ -0,0 +1,283 @@ +"""Turn the weekly live-test run into GitHub issues, one per data source (ยง8). + +The scheduled live tier failed four weeks running in September and October +2026 and nobody looked: a NASA CMR outage made the run red every week, so the +Yukon API breaking underneath it (29 failed tests on 2026-10-20) was one more +line in a log already known to be red. A single red run hides which provider +broke; one issue per provider does not. + +What it does with one pytest JUnit XML report: + +* every **data source with a failing or erroring live test** gets an issue + labelled ``data-source`` (the label the health check uses), listing the + failing tests grouped by error message. A source still failing a week later + has its issue updated and a comment added, so it surfaces again; +* a source whose tests **pass again** has its issue closed. A source whose + tests all skipped (no credentials) proves nothing and is left alone. + +A test's data source is its file: ``tests/stations/test_yukon_client.py`` is +``yukon``, ``tests/timeseries/test_hls.py`` is ``hls``. + +The planning half is pure: :func:`plan` takes the parsed report and returns +the actions, so the offline tests check it without GitHub. Applying reuses +``report_health``'s GitHub calls under its own marker. + +Usage +----- + python scripts/report_live.py live-results.xml [--repo owner/name] + [--run-url URL] [--dry-run] + +``GITHUB_TOKEN`` and ``GITHUB_REPOSITORY`` come from the Actions environment. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import os +import sys +import xml.etree.ElementTree as ET +from collections import Counter +from dataclasses import dataclass, field +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +_spec = importlib.util.spec_from_file_location( + "report_health", _HERE / "report_health.py" +) +report_health = importlib.util.module_from_spec(_spec) +sys.modules.setdefault("report_health", report_health) +_spec.loader.exec_module(report_health) + +Action = report_health.Action + +#: Marker namespace, distinct from the health check's ``esd-health``. +MARKER_PREFIX = "esd-live" +MARKER = "" + +REPRODUCE = "pixi run -e test-py313 pytest {path} -m live -v --tb=short" + + +@dataclass +class Outcome: + """The live tests of one data source in one run.""" + + source: str + path: str + passed: list[str] = field(default_factory=list) + failed: list[tuple[str, str]] = field(default_factory=list) # (test, message) + skipped: list[str] = field(default_factory=list) + + +def source_of(classname: str, file: str | None = None) -> tuple[str, str]: + """Map a JUnit test case to ``(source, test file path)``.""" + if file: + path = file + else: + parts = classname.split(".") + module = next( + (i for i, part in enumerate(parts) if part.startswith("test_")), + len(parts) - 1, + ) + path = "/".join(parts[: module + 1]) + ".py" + stem = Path(path).stem.removeprefix("test_").removesuffix("_client") + return stem, path + + +def _message(node: ET.Element) -> str: + """The first line of a failure, compact enough for a table cell.""" + text = node.get("message") or (node.text or "").strip() + first = text.splitlines()[0] if text else node.tag + return first.strip()[:240] + + +def parse(xml_text: str) -> dict[str, Outcome]: + """Read a pytest JUnit XML report into one :class:`Outcome` per source.""" + outcomes: dict[str, Outcome] = {} + for case in ET.fromstring(xml_text).iter("testcase"): + source, path = source_of(case.get("classname", ""), case.get("file")) + outcome = outcomes.setdefault(source, Outcome(source, path)) + name = case.get("name", "?") + bad = case.find("failure") + if bad is None: + bad = case.find("error") + if bad is not None: + outcome.failed.append((name, _message(bad))) + elif case.find("skipped") is not None: + outcome.skipped.append(name) + else: + outcome.passed.append(name) + return outcomes + + +def _body(outcome: Outcome, run_url: str | None, checked: str) -> str: + total = len(outcome.failed) + len(outcome.passed) + lines = [ + MARKER.format(key=outcome.source), + "", + f"The weekly live tests failed for **`{outcome.source}`**: " + f"{len(outcome.failed)} of {total} run tests in `{outcome.path}`" + + (f" ({len(outcome.skipped)} skipped)." if outcome.skipped else "."), + "", + "| tests | error |", + "| --- | --- |", + ] + for message, count in Counter(m for _, m in outcome.failed).most_common(): + cell = message.replace("|", "\\|") + lines.append(f"| {count} | `{cell}` |") + lines += ["", "
Failing tests", ""] + lines += [f"- `{name}`" for name, _ in outcome.failed] + lines += [ + "", + "
", + "", + f"Run of {checked}" + + (f": {run_url}" if run_url else "") + + ". Each weekly run updates this issue and closes it when the tests " + "pass again. An error naming the provider's own server (HTTP 5xx, an " + "outage page) is usually theirs; an assertion or a parse error is " + "usually a change in what the provider returns.", + "", + "Reproduce locally:", + "", + "```bash", + REPRODUCE.format(path=outcome.path), + "```", + ] + return "\n".join(lines) + + +def plan( + outcomes: dict[str, Outcome], *, run_url: str | None = None, checked: str = "" +) -> list[Action]: + """Work out what this run should do to the tracker.""" + actions: list[Action] = [] + for source, outcome in sorted(outcomes.items()): + if outcome.failed: + actions.append( + Action( + kind="open", + key=source, + title=( + f"Live tests failing: {source} " + f"({len(outcome.failed)} test" + f"{'s' if len(outcome.failed) != 1 else ''})" + ), + body=_body(outcome, run_url, checked), + comment=f"Still failing in the run of {checked}" + + (f": {run_url}" if run_url else "."), + ) + ) + elif outcome.passed: + actions.append( + Action( + kind="close", + key=source, + comment=( + f"Recovered: {len(outcome.passed)} live test" + f"{'s' if len(outcome.passed) != 1 else ''} passed in " + f"the run of {checked}. Closing; the weekly run reopens " + "this if it breaks again." + ), + ) + ) + return actions + + +def summary(outcomes: dict[str, Outcome]) -> str: + """A Markdown table for the job summary: failing sources first.""" + rows = sorted(outcomes.values(), key=lambda o: (not o.failed, o.source)) + lines = [ + "### Live tests by data source", + "", + "| source | failed | passed | skipped | most common error |", + "| --- | --- | --- | --- | --- |", + ] + for o in rows: + top = Counter(m for _, m in o.failed).most_common(1) + error = f"`{top[0][0][:120]}`".replace("|", "\\|") if top else "" + mark = "๐Ÿ”ด " if o.failed else "" + lines.append( + f"| {mark}{o.source} | {len(o.failed)} | {len(o.passed)} | " + f"{len(o.skipped)} | {error} |" + ) + return "\n".join(lines) + + +def apply(actions: list[Action], github, *, log=print) -> list[str]: + """Carry out *actions*; unlike the health check, a repeat open also comments.""" + github.ensure_label() + existing = {} + for issue in github.open_issues(): + body = issue.get("body") or "" + tag = f"", 1)[0].strip()] = issue + + done = [] + for action in actions: + issue = existing.get(action.key) + if action.kind == "open" and issue is None: + created = github.create(action) + done.append(f"opened #{created['number']}: {action.title}") + elif action.kind == "open": + github.update(issue["number"], title=action.title, body=action.body) + github.comment(issue["number"], action.comment) + done.append(f"updated #{issue['number']}: {action.title}") + elif action.kind == "close" and issue is not None: + github.comment(issue["number"], action.comment) + github.update(issue["number"], state="closed", state_reason="completed") + done.append(f"closed #{issue['number']}: live tests for {action.key}") + for line in done: + log(line) + if not done: + log("No issue changes.") + return done + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("report", help="pytest --junitxml output") + parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY", "")) + parser.add_argument("--run-url", default=None) + parser.add_argument("--checked", default="", help="date of the run, for the text") + parser.add_argument( + "--summary", + default=os.environ.get("GITHUB_STEP_SUMMARY"), + help="append the per-source table here (the Actions job summary)", + ) + parser.add_argument( + "--dry-run", action="store_true", help="print the actions, change nothing" + ) + args = parser.parse_args(argv) + + path = Path(args.report) + if not path.exists(): + print(f"No report at {path}; nothing to report.") + return 0 + outcomes = parse(path.read_text()) + table = summary(outcomes) + print(table) + if args.summary: + with open(args.summary, "a", encoding="utf-8") as fh: + fh.write(table + "\n") + + actions = plan(outcomes, run_url=args.run_url, checked=args.checked) + if args.dry_run or not os.environ.get("GITHUB_TOKEN"): + for action in actions: + if action.kind == "open": + print(f"[open] {action.title}") + closes = sum(a.kind == "close" for a in actions) + print(f"[close] {closes} passing sources (closes any open issue)") + if not args.dry_run: + print("(GITHUB_TOKEN is not set, so nothing was sent.)") + return 0 + if not args.repo: + print("--repo or GITHUB_REPOSITORY is required.", file=sys.stderr) + return 2 + apply(actions, report_health.GitHub(args.repo, os.environ["GITHUB_TOKEN"])) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_live_reporting.py b/tests/test_live_reporting.py new file mode 100644 index 00000000..c19a005c --- /dev/null +++ b/tests/test_live_reporting.py @@ -0,0 +1,138 @@ +"""The live-test-to-issue planner (scripts/report_live.py). + +Offline: the report is a JUnit XML string and GitHub is a fake, so the +grouping by data source, the open/update/close paths and the "all skipped +proves nothing" rule are tested without a network. +""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +@pytest.fixture(scope="module") +def live(): + """scripts/report_live.py, imported by path (scripts/ is not a package).""" + spec = importlib.util.spec_from_file_location( + "report_live", REPO_ROOT / "scripts" / "report_live.py" + ) + module = importlib.util.module_from_spec(spec) + sys.modules["report_live"] = module + spec.loader.exec_module(module) + return module + + +# Shaped like the 2026-10-20 run: Yukon broken, CMR down, AWDB fine, Planet +# skipped for want of a key. +REPORT = """ + + + trace + + + trace + + + trace + + + + trace + + + + + + +""" + + +def test_tests_are_grouped_by_their_file(live): + outcomes = live.parse(REPORT) + assert set(outcomes) == {"yukon", "hls", "awdb", "planetscope"} + yukon = outcomes["yukon"] + assert yukon.path == "tests/stations/test_yukon_client.py" + assert len(yukon.failed) == 3 # a failure and an error both count + assert yukon.passed == ["test_variables_required_fields"] + assert outcomes["planetscope"].skipped == ["test_live_search"] + + +def test_a_junit_file_attribute_wins_over_the_classname(live): + assert live.source_of("x.y", "tests/stations/test_nve_client.py") == ( + "nve", + "tests/stations/test_nve_client.py", + ) + + +def test_failing_sources_open_passing_close_skipped_are_left_alone(live): + actions = live.plan(live.parse(REPORT), checked="2026-10-20") + by_key = {a.key: a for a in actions} + assert by_key["yukon"].kind == "open" + assert by_key["yukon"].title == "Live tests failing: yukon (3 tests)" + assert by_key["hls"].title == "Live tests failing: hls (1 test)" + assert by_key["awdb"].kind == "close" + assert "planetscope" not in by_key # all skipped proves nothing + + +def test_the_body_groups_identical_errors(live): + body = live.plan(live.parse(REPORT))[2].body # sorted: awdb, hls, yukon + assert body.startswith("") + assert "| 2 | `YukonError: HTTP 500" in body + assert "| 1 | `assert 0 > 100` |" in body + assert "pytest tests/stations/test_yukon_client.py -m live" in body + + +class FakeGitHub: + def __init__(self, issues): + self.issues = issues + self.calls = [] + + def ensure_label(self): + pass + + def open_issues(self): + return self.issues + + def create(self, action): + self.calls.append(("create", action.key)) + return {"number": 99} + + def update(self, number, **fields): + self.calls.append(("update", number, fields.get("state", "edit"))) + return {} + + def comment(self, number, body): + self.calls.append(("comment", number)) + return {} + + +def test_apply_opens_updates_and_closes_by_marker(live): + issues = [ + {"number": 5, "body": "\nold"}, + {"number": 6, "body": "\nold"}, + # The health check's issue for the same source is a different issue. + {"number": 7, "body": "\nold"}, + ] + github = FakeGitHub(issues) + live.apply(live.plan(live.parse(REPORT)), github, log=lambda _: None) + assert ("create", "hls") in github.calls # not merged into #7 + assert ("update", 5, "edit") in github.calls + assert ("comment", 5) in github.calls # a repeat failure surfaces again + assert ("update", 6, "closed") in github.calls + + +def test_main_without_a_token_only_prints(live, tmp_path, monkeypatch, capsys): + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + report = tmp_path / "live.xml" + report.write_text(REPORT) + summary = tmp_path / "summary.md" + assert live.main([str(report), "--summary", str(summary)]) == 0 + out = capsys.readouterr().out + assert "[open] Live tests failing: yukon (3 tests)" in out + assert "| ๐Ÿ”ด hls | 1 | 0 | 0 |" in summary.read_text() From eb7b52574b2797266964b338c791734188f55bf9 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 07:28:12 +0000 Subject: [PATCH 2/3] chore(yukon): watch the AquaCache API and schema repos; fix the licence link Add path-filtered commit feeds for YGwater's API (inst/plumber on dev.WRB) and AquaCache's schema patches (inst/patches on dev.g) to the upstream watch: the publicly_visible -> default_show rename was public there a week before the client broke. Replace the catalog's OGL-Yukon reference, which returns 404, with the licence URL the Open Yukon dataset pages link to. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqF35dgJEZwc35XaU8n4iu --- WATCHLIST.toml | 19 +++++++++++++++++++ easysnowdata/stations/_catalog.py | 3 ++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/WATCHLIST.toml b/WATCHLIST.toml index 9c8af8e9..fbd1cbd5 100644 --- a/WATCHLIST.toml +++ b/WATCHLIST.toml @@ -526,6 +526,25 @@ id = "feed-geoboundaries-releases" url = "https://github.com/wmgeolab/geoBoundaries/releases.atom" products = ["countries", "states-provinces", "admin-boundaries"] +# Yukon AquaCache: the API and the database schema live on GitHub, and their +# development branches run ahead of `main` and of the deployed service's docs. +# AquaCache patch 65 ("patch_65 live", 2026-10-15) renamed the /timeseries +# column `publicly_visible` to `default_show`; the commit was public a week +# before the client broke. These are path-filtered commit feeds on the branches +# that matched the live API on 2026-10-26 (YGwater `dev.WRB`, AquaCache +# `dev.g`). The branch names change: a feed reported unreachable here means +# check https://github.com/YukonWRB/YGwater/branches and +# https://github.com/YukonWRB/AquaCache/branches and update the URL. +[[feed]] +id = "feed-yukon-ygwater-api-commits" +url = "https://github.com/YukonWRB/YGwater/commits/dev.WRB/inst/plumber.atom" +products = ["yukon-stations", "snow-station-archive"] + +[[feed]] +id = "feed-yukon-aquacache-patch-commits" +url = "https://github.com/YukonWRB/AquaCache/commits/dev.g/inst/patches.atom" +products = ["yukon-stations", "snow-station-archive"] + # โ”€โ”€ Discovery โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ # Everything above watches something the package already uses. These search for # what it does not use yet (scripts/watch.py, check_discover): each run lists a diff --git a/easysnowdata/stations/_catalog.py b/easysnowdata/stations/_catalog.py index 8e196dbe..d97dacde 100644 --- a/easysnowdata/stations/_catalog.py +++ b/easysnowdata/stations/_catalog.py @@ -311,7 +311,8 @@ def _nve_probe() -> None: "https://service.yukon.ca/water-data/api/v2/__docs__/", "https://service.yukon.ca/water-data/shiny/?page=home&lang=en", "https://yukon.ca/en/snow-survey-bulletin", - "https://open.yukon.ca/node/15612", + "https://yukon.ca/en/your-government/open-government/" + "open-government-licence-yukon", ), tags=("swe", "snow depth", "stations", "yukon", "snow course"), ) From ef269502ddcd158f30a758b613f6883960ef3f13 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 07:35:07 +0000 Subject: [PATCH 3/3] fix(ci): read and write the live report as UTF-8 so Windows decodes it test_main_without_a_token_only_prints read the job-summary file with the locale encoding, which on Windows (cp1252) garbles the red-circle marker the script writes as UTF-8. The script's JUnit read had the same flaw. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqF35dgJEZwc35XaU8n4iu --- scripts/report_live.py | 2 +- tests/test_live_reporting.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/report_live.py b/scripts/report_live.py index f1a530a0..2d6b26d0 100644 --- a/scripts/report_live.py +++ b/scripts/report_live.py @@ -255,7 +255,7 @@ def main(argv: list[str] | None = None) -> int: if not path.exists(): print(f"No report at {path}; nothing to report.") return 0 - outcomes = parse(path.read_text()) + outcomes = parse(path.read_text(encoding="utf-8")) table = summary(outcomes) print(table) if args.summary: diff --git a/tests/test_live_reporting.py b/tests/test_live_reporting.py index c19a005c..839c6b5c 100644 --- a/tests/test_live_reporting.py +++ b/tests/test_live_reporting.py @@ -130,9 +130,9 @@ def test_apply_opens_updates_and_closes_by_marker(live): def test_main_without_a_token_only_prints(live, tmp_path, monkeypatch, capsys): monkeypatch.delenv("GITHUB_TOKEN", raising=False) report = tmp_path / "live.xml" - report.write_text(REPORT) + report.write_text(REPORT, encoding="utf-8") summary = tmp_path / "summary.md" assert live.main([str(report), "--summary", str(summary)]) == 0 out = capsys.readouterr().out assert "[open] Live tests failing: yukon (3 tests)" in out - assert "| ๐Ÿ”ด hls | 1 | 0 | 0 |" in summary.read_text() + assert "| ๐Ÿ”ด hls | 1 | 0 | 0 |" in summary.read_text(encoding="utf-8")