From c1ef3ca539fddfd56322449fc0759a77a1308d16 Mon Sep 17 00:00:00 2001
From: Marius Ciepluch <11855163+norandom@users.noreply.github.com>
Date: Sun, 6 Sep 2026 14:20:52 +0200
Subject: [PATCH] fix(learning-harness): a provider hiccup costs one pair, not
 the whole run

Round zero is the most expensive thing this harness does. An error two
thirds of the way through raised straight out of `run_baseline` and took
everything already measured and paid for with it, so the only honest
response to a flaky endpoint was to run the whole thing again and hope. The
pair is now recorded as unscorable with the reason `detector_unreachable`
and the run finishes.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
---
 src/openultrasast/learning/rounds.py | 17 +++++++++++++-
 tests/test_learning_rounds.py        | 34 ++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/src/openultrasast/learning/rounds.py b/src/openultrasast/learning/rounds.py
index 0971d39..6f6e482 100644
--- a/src/openultrasast/learning/rounds.py
+++ b/src/openultrasast/learning/rounds.py
@@ -123,7 +123,22 @@ def run_baseline(
         config = configs.get(name)
         if config is None:
             continue  # a family with no configuration is not measured rather than measured with someone else's
-        scores.append(score_case(case, scan_factory(config), taxonomy=taxonomy, runs=k_runs, family=name))
+        try:
+            scores.append(score_case(case, scan_factory(config), taxonomy=taxonomy, runs=k_runs, family=name))
+        except Exception:  # noqa: BLE001 — a provider hiccup costs one pair, not the whole run
+            # Round zero is the most expensive thing this harness does. Raising here threw away everything already
+            # measured and paid for, so the only honest response to a flaky endpoint was to run it all again.
+            scores.append(
+                PairFamilyScore(
+                    pair=case.name,
+                    family=name,
+                    slice=case.slice,
+                    fix_date=case.fix_date,
+                    runs=(),
+                    outcome="unscorable",
+                    unscorable_reason="detector_unreachable",
+                )
+            )
     metrics = aggregate(scores, taxonomy=taxonomy, cutoff=cutoff)
     floors = {name: _floor(name, block, [item for item in scores if item.family == name], k_runs) for name, block in metrics.items()}
     report = BaselineReport(
diff --git a/tests/test_learning_rounds.py b/tests/test_learning_rounds.py
index 1405288..be0a148 100644
--- a/tests/test_learning_rounds.py
+++ b/tests/test_learning_rounds.py
@@ -224,3 +224,37 @@ def test_round_zero_records_what_it_spent(tmp_path: Path) -> None:
     )
     assert report.cost_usd == 1.25
     assert json.loads((tmp_path / "out" / "baseline" / "m" / "report.json").read_text())["cost_usd"] == 1.25
+
+
+def test_round_zero_records_the_pairs_it_could_not_reach_instead_of_losing_the_run(tmp_path: Path) -> None:
+    """One transient error must not cost a run that has already paid for hundreds of calls.
+
+    Round zero is the most expensive thing this harness does. A provider hiccup two thirds of the way through used
+    to raise straight out of `run_baseline`, and everything measured so far went with it — so the honest response
+    to a flaky endpoint was to run the whole thing again and hope."""
+    from openultrasast.learning.rounds import run_baseline
+
+    taxonomy = load_families()
+    seen: list[str] = []
+
+    def build(_config):  # type: ignore[no-untyped-def]
+        def scan(root: Path) -> list[StaticFinding]:
+            seen.append(root.name)
+            if len(seen) == 3:
+                raise TimeoutError("the endpoint hung up")
+            return _steady(root)
+
+        return scan
+
+    report = run_baseline(
+        [_case(tmp_path, "a"), _case(tmp_path, "b")],
+        taxonomy=taxonomy,
+        configs_dir=tmp_path / "configs",
+        scan_factory=build,
+        model="m",
+        out_dir=tmp_path / "out",
+        k_runs=3,
+    )
+    assert report.metrics["injection"]["scorable"] == 1, "the pair that was reached is still measured"
+    assert report.metrics["injection"]["unscorable"] == {"detector_unreachable": 1}
+    assert (tmp_path / "out" / "baseline" / "m" / "report.json").is_file()