Claude/focused carson 2fjr5w

README.md

@@ -94,10 +94,10 @@ Then:
 - **Stop it**: `docker compose down` keeps the results (in the `radiator-db`
   volume) for next time; `docker compose down -v` wipes them.
 
-If the harness exits with `Permission denied: '/app/logs/harness.log'` (a Linux
-host where `./logs` isn't writable by the container's user), run
-`chmod a+w logs` (and remove `logs/harness.log` if it exists and isn't
-yours), then `docker compose up` again.
+If the harness warns `Can't write ./logs/harness.log` (a Linux host where
+`./logs` isn't writable by the container's user), it logs to the console only;
+to get the file back, run `chmod a+w logs` (and remove `logs/harness.log` if it
+exists and isn't yours), then `docker compose up` again.
 
 #### Reporting to a radiator from the harness
 Point the harness at a radiator with `RADIATOR_URL` and `RADIATOR_TOKEN` (or

deploy/helm/README.md

@@ -93,6 +93,20 @@ kubectl -n translator create job --from=cronjob/test-harness-cycle-ci cycle-ci-n
 kubectl -n translator create job --from=cronjob/test-harness-weekly-digest digest-now
 ```
 
+That runs the whole cycle. To run one step of it (acceptance, pathfinder or
+performance) on its own, with the suites and targets from that environment's
+CronJob:
+
+```
+deploy/run-step.sh ci acceptance
+deploy/run-step.sh prod performance -n translator
+deploy/run-step.sh test pathfinder --dry-run   # print the Job instead
+```
+
+It needs `kubectl` and `jq`, and won't start while a Job of the same CronJob is
+still running (a Job created by hand ignores the CronJob's
+`concurrencyPolicy`); `--force` starts it anyway. `--help` lists the rest.
+
 `suspend: true` on an environment (or `digest.suspend`, `sweep.suspend`) keeps
 its CronJob but stops it from running.
 

deploy/helm/test-harness/templates/NOTES.txt

@@ -12,3 +12,10 @@ Scheduled Test Harness runs ({{ .Values.timeZone }}):
 
 Run one now, eg:
   kubectl -n {{ .Release.Namespace }} create job --from=cronjob/{{ include "harness.fullname" . }}-weekly-digest digest-now
+{{- with $cycles }}
+{{- with first . }}
+
+Run one step of a cycle (acceptance, pathfinder or performance), eg:
+  deploy/run-step.sh {{ .name }} acceptance -n {{ $.Release.Namespace }}{{ if ne (include "harness.fullname" $) "test-harness" }} --cronjob {{ include "harness.fullname" $ }}-cycle-{{ .name }}{{ end }}
+{{- end }}
+{{- end }}

deploy/run-step.sh

@@ -0,0 +1,140 @@
+#!/usr/bin/env bash
+# Run one step of an environment's cycle now, instead of the whole cycle.
+#
+#   deploy/run-step.sh ci acceptance
+#   deploy/run-step.sh prod performance -n translator
+#   deploy/run-step.sh test pathfinder --dry-run      # print the Job, don't create it
+#
+# The Job is copied from the environment's cycle CronJob, so it has the same
+# image, secrets, Slack and radiator settings, resources and deadline; only the
+# arguments change, to that step alone with the CronJob's own suite (and, for
+# performance, its targets and profile). The step reports as usual and the
+# cycle posts its board for the one step.
+#
+# A Job created by hand ignores the CronJob's concurrencyPolicy, so this
+# refuses to start while a Job of the same CronJob is still running (--force
+# to start anyway).
+#
+# Needs kubectl and jq.
+
+set -euo pipefail
+
+usage() {
+  cat <<EOF
+Usage: $(basename "$0") ENVIRONMENT STEP [options]
+
+  ENVIRONMENT   a cycles.environments name, eg dev, ci, test, prod
+  STEP          acceptance, pathfinder or performance
+
+Options:
+  -n, --namespace NS   the namespace (default: \$NAMESPACE, else translator-dev)
+  --cronjob NAME       the CronJob to copy (default: test-harness-cycle-ENVIRONMENT)
+  --name NAME          the Job's name (default: STEP-ENVIRONMENT-<timestamp>)
+  --dry-run            print the Job instead of creating it
+  --force              start even if a Job of the same CronJob is running
+  -h, --help           show this help
+EOF
+}
+
+die() {
+  echo "$(basename "$0"): $*" >&2
+  exit 1
+}
+
+namespace="${NAMESPACE:-translator-dev}"
+cronjob=""
+job_name=""
+dry_run=false
+force=false
+positional=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -n | --namespace) namespace="${2:?$1 needs a value}"; shift 2 ;;
+    --cronjob) cronjob="${2:?$1 needs a value}"; shift 2 ;;
+    --name) job_name="${2:?$1 needs a value}"; shift 2 ;;
+    --dry-run) dry_run=true; shift ;;
+    --force) force=true; shift ;;
+    -h | --help) usage; exit 0 ;;
+    -*) usage >&2; die "unknown option $1" ;;
+    *) positional+=("$1"); shift ;;
+  esac
+done
+
+[[ ${#positional[@]} -eq 2 ]] || { usage >&2; exit 2; }
+env_name="${positional[0]}"
+step="${positional[1]}"
+case "$step" in
+  acceptance | pathfinder | performance) ;;
+  *) die "STEP must be acceptance, pathfinder or performance, not '$step'" ;;
+esac
+
+for tool in kubectl jq; do
+  command -v "$tool" >/dev/null || die "needs $tool"
+done
+
+cronjob="${cronjob:-test-harness-cycle-$env_name}"
+job_name="${job_name:-$step-$env_name-$(date +%Y%m%d-%H%M%S)}"
+kc=(kubectl -n "$namespace")
+
+"${kc[@]}" get cronjob "$cronjob" >/dev/null ||
+  die "no CronJob $cronjob in namespace $namespace (see --cronjob, -n)"
+
+if ! $dry_run && ! $force; then
+  running=$("${kc[@]}" get jobs -o json | jq -r --arg cj "$cronjob" '
+    .items[]
+    | select(any(.metadata.ownerReferences[]?; .kind == "CronJob" and .name == $cj))
+    | select((.status.active // 0) > 0)
+    | .metadata.name')
+  if [[ -n "$running" ]]; then
+    die "a Job of $cronjob is still running ($(echo "$running" | paste -sd, -)):" \
+      "two runs at once would load the same services together. Wait for it," \
+      "or pass --force."
+  fi
+fi
+
+# The cycle's arguments with every other step left out: the suite source and
+# output flags, this step's flag (with, for performance, its targets and
+# profile), and everything after the `--` as it is.
+job=$("${kc[@]}" create job "$job_name" --from="cronjob/$cronjob" --dry-run=client -o json |
+  jq --arg step "$step" --arg env "$env_name" '
+    def keep($flag):
+      ["--download", "--tests_url", "--tests_dir", "--output_dir", "--\($step)"]
+      + (if $step == "performance"
+         then ["--performance_target", "--performance_profile"] else [] end)
+      | index([$flag]) != null;
+    def only_step:
+      (index("--") // length) as $sep
+      | (.[:$sep] | reduce .[] as $arg ({out: [], value: null};
+          if .value != null then
+            # the value of the flag before it
+            (if .value then .out += [$arg] else . end) | .value = null
+          else
+            ($arg | split("=")[0]) as $flag
+            | (keep($flag)) as $k
+            | (if $k then .out += [$arg] else . end)
+            # --download takes no value; --flag=value carries its own
+            | if $flag == "--download" or ($arg | contains("=")) then .
+              else .value = $k end
+          end
+        ) | .out) + .[$sep:];
+    .spec.template.spec.containers[0].args |= only_step
+    | .metadata.labels["translator.ncats.io/environment"] = $env
+    | .metadata.labels["translator.ncats.io/step"] = $step
+  ')
+
+args=$(jq -c '.spec.template.spec.containers[0].args' <<<"$job")
+jq -e --arg flag "--$step" 'index([$flag]) != null or any(.[]; startswith($flag + "="))' \
+  <<<"$args" >/dev/null ||
+  die "$cronjob has no $step step (eg performance: \"\" in its values)"
+
+if $dry_run; then
+  echo "$job"
+  exit 0
+fi
+
+echo "Starting $step on $env_name: job/$job_name"
+echo "  test-harness-cycle $(jq -r 'join(" ")' <<<"$args")"
+"${kc[@]}" apply -f - <<<"$job" >/dev/null
+echo "Follow it with:"
+echo "  kubectl -n $namespace logs -f job/$job_name"

setup.py

@@ -7,7 +7,7 @@
 
 setup(
     name="sri-test-harness",
-    version="0.8.1",
+    version="0.8.2",
     author="Max Wang",
     author_email="max@covar.com",
     url="https://github.com/TranslatorSRI/TestHarness",

test_harness/logger.py

@@ -1,7 +1,9 @@
 from collections import deque
+from copy import deepcopy
 from datetime import datetime
 import logging
 from logging import config
+import os
 
 logging_config = {
     "version": 1,
@@ -96,7 +98,20 @@ def get_logger(qid, log_level):
 
 
 def setup_logger():
-    config.dictConfig(logging_config)
+    log_file = logging_config["handlers"]["file"]["filename"]
+    try:
+        # the image leaves logs/ out (.dockerignore), so make it here
+        os.makedirs(os.path.dirname(log_file), exist_ok=True)
+        config.dictConfig(logging_config)
+    except (OSError, ValueError) as e:
+        # an unwritable log file shouldn't stop a run: log to the console only
+        console_only = deepcopy(logging_config)
+        del console_only["handlers"]["file"]
+        console_only["root"]["handlers"] = ["console"]
+        config.dictConfig(console_only)
+        logging.getLogger(__name__).warning(
+            f"Can't write {log_file} ({e.__cause__ or e}); logging to the console only."
+        )
     logging.getLogger("httpcore").setLevel(logging.WARNING)
     logging.getLogger("httpx").setLevel(logging.WARNING)
     logging.getLogger("urllib3").setLevel(logging.WARNING)

test_harness/trapi.py

@@ -73,6 +73,7 @@ def binding_ids(binding: Any) -> List[str]:
 
 # --- query parameters ----------------------------------------------------------
 
+
 def parse_query_parameters(text: Optional[str]) -> Optional[Dict[str, Any]]:
     """Read query parameters given on the command line or in the environment.
 

tests/test_logger.py

@@ -0,0 +1,34 @@
+"""setup_logger runs when the harness is imported, so it must not crash on the log file."""
+
+import logging
+
+import pytest
+
+from test_harness.logger import setup_logger
+
+
+@pytest.fixture(autouse=True)
+def restore_logging():
+    root = logging.getLogger()
+    handlers, level = root.handlers[:], root.level
+    yield
+    for handler in root.handlers:
+        if handler not in handlers:
+            handler.close()
+    root.handlers, root.level = handlers, level
+
+
+def test_makes_the_log_directory(tmp_path, monkeypatch):
+    # the image has no logs/ (.dockerignore leaves it out)
+    monkeypatch.chdir(tmp_path)
+    setup_logger()
+    assert (tmp_path / "logs" / "harness.log").exists()
+
+
+def test_falls_back_to_the_console(tmp_path, monkeypatch, capsys):
+    monkeypatch.chdir(tmp_path)
+    (tmp_path / "logs").write_text("not a directory")
+    setup_logger()
+    assert [type(h) for h in logging.getLogger().handlers] == [logging.StreamHandler]
+    # dictConfig replaces caplog's handler; the console handler writes to stderr
+    assert "logging to the console only" in capsys.readouterr().err

tests/test_query_parameters.py

@@ -159,7 +159,9 @@ def _main_args(tmp_path, **extra):
 def test_main_refuses_bad_parameters_before_running(mocker, tmp_path):
     run = mocker.patch("test_harness.main.run_tests")
     code = main(
-        _main_args(tmp_path, trapi_version="1.6.0", query_parameters={"timeout": "soon"})
+        _main_args(
+            tmp_path, trapi_version="1.6.0", query_parameters={"timeout": "soon"}
+        )
     )
     assert code == 1
     run.assert_not_called()