diff --git a/README.md b/README.md index e9f4050..714d70f 100644 --- a/README.md +++ b/README.md @@ -401,9 +401,9 @@ sweep: ```bash test-harness-sweep --suite performance_tests --env ci --download \ ars=https://ars.ci.transltr.io \ - aragorn=https://aragorn.ci.transltr.io \ - arax=https://arax.ci.transltr.io \ - bte=https://bte.ci.transltr.io \ + shepherd-aragorn=https://aragorn.ci.transltr.io \ + shepherd-arax=https://arax.ci.transltr.io \ + shepherd-bte=https://bte.ci.transltr.io \ -- --performance_profile mixed ``` @@ -411,6 +411,14 @@ Each `NAME=URL` pair becomes one `test-harness --env ENV --target_url URL --targ invocation, in the order given, and the sweep waits for each to finish before starting the next. Anything after `--` is passed through to every invocation. +`NAME` must be the infores the service is registered under in SmartAPI. The +Shepherd-hosted ARAs are `infores:shepherd-aragorn`, `infores:shepherd-arax` and +`infores:shepherd-bte`, not the bare `aragorn`/`arax`/`bte`. `--trapi_version +auto` (the default) looks the TRAPI version up by that name, and an unregistered +name falls back to 2.0.0 with only a log warning. `NAME` also labels the +results (each target's output directory and filename prefix, and the agent the +Information Radiator reports them under). + Pass `--tests_url` (and `--tests_dir`) to the **sweep**, not through the `--`. They belong to the harness's `download`/`load` subcommands, and passthrough arguments go in front of the subcommand, where argparse rejects them; the sweep diff --git a/deploy/cronjob.example.yaml b/deploy/cronjob.example.yaml index 624b443..6ef4942 100644 --- a/deploy/cronjob.example.yaml +++ b/deploy/cronjob.example.yaml @@ -97,9 +97,9 @@ spec: - name: PERFORMANCE_TARGETS value: >- ars=https://ars.ci.transltr.io, - aragorn=https://aragorn.ci.transltr.io, - arax=https://arax.ci.transltr.io, - bte=https://bte.ci.transltr.io + shepherd-aragorn=https://aragorn.ci.transltr.io, + shepherd-arax=https://arax.ci.transltr.io, + shepherd-bte=https://bte.ci.transltr.io # Without this, Python buffers stdout when it isn't a terminal # and `kubectl logs` shows nothing for hours. diff --git a/deploy/helm/test-harness/templates/sweep.yaml b/deploy/helm/test-harness/templates/sweep.yaml index b7d7f8f..169f8aa 100644 --- a/deploy/helm/test-harness/templates/sweep.yaml +++ b/deploy/helm/test-harness/templates/sweep.yaml @@ -1,6 +1,6 @@ {{- if .Values.sweep.enabled }} {{- if not .Values.sweep.targets }} -{{- fail "sweep.targets is required when the sweep is enabled, eg [\"aragorn=https://aragorn.ci.transltr.io\"]" }} +{{- fail "sweep.targets is required when the sweep is enabled, eg [\"shepherd-aragorn=https://aragorn.ci.transltr.io\"]" }} {{- end }} apiVersion: batch/v1 kind: CronJob diff --git a/deploy/helm/test-harness/values.example.yaml b/deploy/helm/test-harness/values.example.yaml index 524920e..66d829c 100644 --- a/deploy/helm/test-harness/values.example.yaml +++ b/deploy/helm/test-harness/values.example.yaml @@ -76,6 +76,6 @@ sweep: schedule: "0 2 * * 0" env: ci targets: - - aragorn=https://aragorn.ci.transltr.io - - arax=https://arax.ci.transltr.io - - bte=https://bte.ci.transltr.io + - shepherd-aragorn=https://aragorn.ci.transltr.io + - shepherd-arax=https://arax.ci.transltr.io + - shepherd-bte=https://bte.ci.transltr.io diff --git a/requirements-runners.txt b/requirements-runners.txt index 11dbc99..c82f306 100644 --- a/requirements-runners.txt +++ b/requirements-runners.txt @@ -7,7 +7,7 @@ # arithmetic. test_harness/performance_test_runner.py only drives its CLI. # Pinned to a release tag (not @main) so the image is reproducible and an # upgrade is a reviewable one-line change; bump it when HelmsDeep tags a release. -helmsdeep @ git+https://github.com/TranslatorSRI/HelmsDeep@v0.4.0 +helmsdeep @ git+https://github.com/TranslatorSRI/HelmsDeep@v0.5.0 # HelmsDeep brings locust in; pinned here so the harness's image is # reproducible rather than tracking whatever HelmsDeep's floor resolves to. locust==2.38.1 diff --git a/test_harness/main.py b/test_harness/main.py index 5fa3296..5315a30 100644 --- a/test_harness/main.py +++ b/test_harness/main.py @@ -709,7 +709,9 @@ def cli(): type=str, help=( "The infores identifier of the service at --target_url, with or " - "without the 'infores:' prefix, e.g. aragorn or infores:aragorn. " + "without the 'infores:' prefix, e.g. shepherd-aragorn or " + "infores:shepherd-aragorn -- the one it is registered under in " + "SmartAPI, which --trapi_version auto reads. " "Anything other than ars is queried directly as a single service " "(POST to /query); ars uses the normal ARS " "submit/poll flow. Must be used with --target_url." diff --git a/test_harness/sweep.py b/test_harness/sweep.py index 9943109..faa9d14 100644 --- a/test_harness/sweep.py +++ b/test_harness/sweep.py @@ -21,9 +21,9 @@ test-harness-sweep --suite performance_tests --env ci --download \\ ars=https://ars.ci.transltr.io \\ - aragorn=https://aragorn.ci.transltr.io \\ - arax=https://arax.ci.transltr.io \\ - bte=https://bte.ci.transltr.io \\ + shepherd-aragorn=https://aragorn.ci.transltr.io \\ + shepherd-arax=https://arax.ci.transltr.io \\ + shepherd-bte=https://bte.ci.transltr.io \\ -- --performance_profile mixed Targets can also come from the PERFORMANCE_TARGETS environment variable as a @@ -240,7 +240,7 @@ def cli(argv: Sequence[str] = None) -> int: "harness invocation, e.g.:\n\n" " test-harness-sweep --suite performance_tests --env ci --download \\\n" " ars=https://ars.ci.transltr.io \\\n" - " aragorn=https://aragorn.ci.transltr.io \\\n" + " shepherd-aragorn=https://aragorn.ci.transltr.io \\\n" " -- --performance_profile mixed --log_level INFO\n" ), ) @@ -251,8 +251,11 @@ def cli(argv: Sequence[str] = None) -> int: help=( "The services to run against, in order, e.g. " "ars=https://ars.ci.transltr.io. NAME is the infores identifier " - f"passed to --target. Defaults to the {TARGETS_ENV} environment " - "variable, as a comma-separated list of the same pairs." + "passed to --target, and must be the one the service is registered " + "under in SmartAPI (shepherd-aragorn, not aragorn): --trapi_version " + "auto looks the version up by it. Defaults to the " + f"{TARGETS_ENV} environment variable, as a comma-separated list of " + "the same pairs." ), ) parser.add_argument(